You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ve...@apache.org on 2015/06/17 01:04:35 UTC
[01/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 9071ce2a3 -> 7f2b49ba8
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryJerseyResourceIT.java
deleted file mode 100755
index 0682295..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryJerseyResourceIT.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.collect.ImmutableList;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import java.util.List;
-
-/**
- * Search Integration Tests.
- */
-public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
-
- createTypes();
- createInstance();
- }
-
- @Test
- public void testSearchByDSL() throws Exception {
- String dslQuery = "from dsl_test_type";
- WebResource resource = service
- .path("api/metadata/discovery/search/dsl")
- .queryParam("query", dslQuery);
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- Assert.assertEquals(response.getString("query"), dslQuery);
- Assert.assertEquals(response.getString("queryType"), "dsl");
-
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(results);
-
- JSONArray rows = results.getJSONArray(MetadataServiceClient.ROWS);
- Assert.assertEquals(rows.length(), 1);
-
- int numRows = response.getInt(MetadataServiceClient.COUNT);
- Assert.assertEquals(numRows, 1);
-
- }
-
- @Test
- public void testSearchByDSLForUnknownType() throws Exception {
- String dslQuery = "from blah";
- WebResource resource = service
- .path("api/metadata/discovery/search/dsl")
- .queryParam("query", dslQuery);
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(),
- Response.Status.BAD_REQUEST.getStatusCode());
- }
-
- @Test
- public void testSearchUsingGremlin() throws Exception {
- String query = "g.V.has('type', 'dsl_test_type').toList()";
- WebResource resource = service
- .path("api/metadata/discovery/search")
- .queryParam("query", query);
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- Assert.assertEquals(response.getString("query"), query);
- Assert.assertEquals(response.getString("queryType"), "gremlin");
- }
-
- @Test
- public void testSearchUsingDSL() throws Exception {
- String query = "from dsl_test_type";
- WebResource resource = service
- .path("api/metadata/discovery/search")
- .queryParam("query", query);
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- Assert.assertEquals(response.getString("query"), query);
- Assert.assertEquals(response.getString("queryType"), "dsl");
- }
-
- @Test(enabled = false)
- public void testSearchUsingFullText() throws Exception {
- String query = "foundation_etl";
- JSONObject response = serviceClient.searchByFullText(query);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- Assert.assertEquals(response.getString("query"), query);
- Assert.assertEquals(response.getString("queryType"), "full-text");
-
- JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertEquals(results.length(), 1);
-
- JSONObject row = results.getJSONObject(0);
- Assert.assertNotNull(row.get("guid"));
- Assert.assertEquals(row.getString("typeName"), "dsl_test_type");
- Assert.assertNotNull(row.get("score"));
-
- int numRows = response.getInt(MetadataServiceClient.COUNT);
- Assert.assertEquals(numRows, 1);
- }
-
- private void createTypes() throws Exception {
- HierarchicalTypeDefinition<ClassType> dslTestTypeDefinition =
- TypesUtil.createClassTypeDef("dsl_test_type",
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
- TypesUtil.createTraitTypeDef("Classification",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef("PII_TYPE", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> phiTrait =
- TypesUtil.createTraitTypeDef("PHI", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> pciTrait =
- TypesUtil.createTraitTypeDef("PCI", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> soxTrait =
- TypesUtil.createTraitTypeDef("SOX", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> secTrait =
- TypesUtil.createTraitTypeDef("SEC", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> financeTrait =
- TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
-
- TypesDef typesDef = TypeUtils.getTypesDef(
- ImmutableList.<EnumTypeDefinition>of(),
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
- soxTrait, secTrait, financeTrait),
- ImmutableList.of(dslTestTypeDefinition));
- createType(typesDef);
- }
-
- private Id createInstance() throws Exception {
- Referenceable entityInstance = new Referenceable("dsl_test_type",
- "Classification", "PII_TYPE", "PHI", "PCI", "SOX", "SEC", "Finance");
- entityInstance.set("name", "foo name");
- entityInstance.set("description", "bar description");
-
- Struct traitInstance = (Struct) entityInstance.getTrait("Classification");
- traitInstance.set("tag", "foundation_etl");
-
- List<String> traits = entityInstance.getTraits();
- Assert.assertEquals(traits.size(), 7);
-
- return createInstance(entityInstance);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/RexsterGraphJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/RexsterGraphJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/RexsterGraphJerseyResourceIT.java
deleted file mode 100755
index 7ebd7e1..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/RexsterGraphJerseyResourceIT.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-
-/**
- * Integration tests for Rexster Graph Jersey Resource.
- */
-@Test
-public class RexsterGraphJerseyResourceIT extends BaseResourceIT {
-
- @BeforeClass
- @Override
- public void setUp() throws Exception {
- super.setUp();
- }
-
- @Test(enabled = false)
- public void testGetVertex() throws Exception {
- // todo: add a vertex before fetching it
-
- WebResource resource = service
- .path("api/metadata/graph/vertices")
- .path("0");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
- String response = clientResponse.getEntity(String.class);
- Assert.assertNotNull(response);
- }
-
- public void testGetVertexWithInvalidId() throws Exception {
- WebResource resource = service
- .path("api/metadata/graph/vertices/blah");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
- }
-
- public void testGetVertexProperties() throws Exception {
-
- }
-
- public void testGetVertices() throws Exception {
-
- }
-
- public void testGetVertexEdges() throws Exception {
-
- }
-
- public void testGetEdge() throws Exception {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java
deleted file mode 100755
index 0d9e710..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/TypesJerseyResourceIT.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.collect.ImmutableList;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization$;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Integration test for types jersey resource.
- */
-public class TypesJerseyResourceIT extends BaseResourceIT {
-
- private List<HierarchicalTypeDefinition> typeDefinitions;
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
-
- typeDefinitions = createHiveTypes();
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- typeDefinitions.clear();
- }
-
- @Test
- public void testSubmit() throws Exception {
- for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
- String typesAsJSON = TypesSerialization.toJson(typeDefinition);
- System.out.println("typesAsJSON = " + typesAsJSON);
-
- WebResource resource = service
- .path("api/metadata/types");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- JSONArray typesAdded = response.getJSONArray(MetadataServiceClient.TYPES);
- Assert.assertEquals(typesAdded.length(), 1);
- Assert.assertEquals(typesAdded.getJSONObject(0).getString("name"), typeDefinition.typeName);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- }
- }
-
- @Test(dependsOnMethods = "testSubmit")
- public void testGetDefinition() throws Exception {
- for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
- System.out.println("typeName = " + typeDefinition.typeName);
-
- WebResource resource = service
- .path("api/metadata/types")
- .path(typeDefinition.typeName);
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.DEFINITION));
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- String typesJson = response.getString(MetadataServiceClient.DEFINITION);
- final TypesDef typesDef = TypesSerialization.fromJson(typesJson);
- List<HierarchicalTypeDefinition<ClassType>> hierarchicalTypeDefinitions = typesDef.classTypesAsJavaList();
- for(HierarchicalTypeDefinition<ClassType> classType : hierarchicalTypeDefinitions) {
- for(AttributeDefinition attrDef : classType.attributeDefinitions) {
- if("name".equals(attrDef.name)) {
- Assert.assertEquals(attrDef.isIndexable, true);
- Assert.assertEquals(attrDef.isUnique, true);
- }
- }
- }
- }
- }
-
- @Test
- public void testGetDefinitionForNonexistentType() throws Exception {
- WebResource resource = service
- .path("api/metadata/types")
- .path("blah");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
- }
-
- @Test(dependsOnMethods = "testSubmit")
- public void testGetTypeNames() throws Exception {
- WebResource resource = service
- .path("api/metadata/types");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(list);
- }
-
- @Test
- public void testGetTraitNames() throws Exception {
- String[] traitsAdded = addTraits();
-
- WebResource resource = service
- .path("api/metadata/types");
-
- ClientResponse clientResponse = resource
- .queryParam("type", DataTypes.TypeCategory.TRAIT.name())
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(list);
- Assert.assertTrue(list.length() >= traitsAdded.length);
- }
-
- private String[] addTraits() throws Exception {
- String[] traitNames = {
- "class_trait",
- "secure_trait",
- "pii_trait",
- "ssn_trait",
- "salary_trait",
- "sox_trait",
- };
-
- for (String traitName : traitNames) {
- HierarchicalTypeDefinition<TraitType> traitTypeDef =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
- String json = TypesSerialization$.MODULE$.toJson(traitTypeDef, true);
- createType(json);
- }
-
- return traitNames;
- }
-
- private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
- ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
-
- HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
- TypesUtil.createClassTypeDef("database",
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
- typeDefinitions.add(databaseTypeDefinition);
-
- HierarchicalTypeDefinition<ClassType> tableTypeDefinition = TypesUtil.createClassTypeDef(
- "table",
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
- new AttributeDefinition("database",
- "database", Multiplicity.REQUIRED, false, "database"));
- typeDefinitions.add(tableTypeDefinition);
-
- HierarchicalTypeDefinition<TraitType> fetlTypeDefinition = TypesUtil.createTraitTypeDef(
- "fetl",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
- typeDefinitions.add(fetlTypeDefinition);
-
- return typeDefinitions;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerIT.java
deleted file mode 100644
index 3c5b229..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerIT.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.service;
-
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase{
- @Test
- public void testServerConfiguredUsingCredentialProvider() throws Exception {
- // setup the configuration
- final PropertiesConfiguration configuration = new PropertiesConfiguration();
- configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
- // setup the credential provider
- setupCredentials();
-
- SecureEmbeddedServer secureEmbeddedServer = null;
- try {
- String appPath = System.getProperty("user.dir") + getWarPath();
- secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) {
- @Override
- protected PropertiesConfiguration getConfiguration() {
- return configuration;
- }
- };
- secureEmbeddedServer.server.start();
-
- URL url = new URL("https://localhost:21443/");
- HttpURLConnection connection = (HttpURLConnection)url.openConnection();
- connection.setRequestMethod("GET");
- connection.connect();
-
- // test to see whether server is up and root page can be served
- Assert.assertEquals(connection.getResponseCode(), 200);
- } finally {
- secureEmbeddedServer.server.stop();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerITBase.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerITBase.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerITBase.java
deleted file mode 100755
index 64358b8..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServerITBase.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.service;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metadata.web.resources.*;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.mortbay.jetty.webapp.WebAppContext;
-import org.testng.Assert;
-import org.testng.TestListenerAdapter;
-import org.testng.TestNG;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.core.UriBuilder;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.nio.file.Files;
-import java.util.List;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-/**
- *
- */
-public class SecureEmbeddedServerITBase {
-
-
- private SecureEmbeddedServer secureEmbeddedServer;
- protected String providerUrl;
- private Path jksPath;
- protected WebResource service;
-
- static {
- //for localhost testing only
- javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
- new javax.net.ssl.HostnameVerifier(){
-
- public boolean verify(String hostname,
- javax.net.ssl.SSLSession sslSession) {
- if (hostname.equals("localhost")) {
- return true;
- }
- return false;
- }
- });
- System.setProperty("javax.net.ssl.trustStore", DEFAULT_KEYSTORE_FILE_LOCATION);
- System.setProperty("javax.net.ssl.trustStorePassword", "keypass");
- System.setProperty("javax.net.ssl.trustStoreType", "JKS");
- }
-
- @BeforeClass
- public void setupServerURI () throws Exception {
- BaseResourceIT.baseUrl = "https://localhost:21443";
- }
-
- @AfterClass
- public void resetServerURI() throws Exception {
- BaseResourceIT.baseUrl = "http://localhost:21000";
- }
-
- @BeforeMethod
- public void setup() throws Exception {
- jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
-
- String baseUrl = "https://localhost:21443/";
-
- DefaultClientConfig config = new DefaultClientConfig();
- Client client = Client.create(config);
- client.resource(UriBuilder.fromUri(baseUrl).build());
-
- service = client.resource(UriBuilder.fromUri(baseUrl).build());
- }
-
- @Test
- public void testNoConfiguredCredentialProvider() throws Exception {
-
- try {
- secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance");
- WebAppContext webapp = new WebAppContext();
- webapp.setContextPath("/");
- webapp.setWar(System.getProperty("user.dir") + getWarPath());
- secureEmbeddedServer.server.setHandler(webapp);
-
- secureEmbeddedServer.server.start();
-
- Assert.fail("Should have thrown an exception");
- } catch (IOException e) {
- Assert.assertEquals("No credential provider path configured for storage of certificate store passwords", e.getMessage());
- } finally {
- secureEmbeddedServer.server.stop();
- }
- }
-
- @Test
- public void testMissingEntriesInCredentialProvider() throws Exception {
- // setup the configuration
- final PropertiesConfiguration configuration = new PropertiesConfiguration();
- configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- try {
- secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
- @Override
- protected PropertiesConfiguration getConfiguration() {
- return configuration;
- }
- };
- Assert.fail("No entries should generate an exception");
- } catch (IOException e) {
- Assert.assertTrue(e.getMessage().startsWith("No credential entry found for"));
- } finally {
- secureEmbeddedServer.server.stop();
- }
-
- }
-
- /**
- * Runs the existing webapp test cases, this time against the initiated secure server instance.
- * @throws Exception
- */
- @Test
- public void runOtherSuitesAgainstSecureServer() throws Exception {
- final PropertiesConfiguration configuration = new PropertiesConfiguration();
- configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
- // setup the credential provider
- setupCredentials();
-
- try {
- secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
- @Override
- protected PropertiesConfiguration getConfiguration() {
- return configuration;
- }
- };
- WebAppContext webapp = new WebAppContext();
- webapp.setContextPath("/");
- webapp.setWar(System.getProperty("user.dir") + getWarPath());
- secureEmbeddedServer.server.setHandler(webapp);
-
- secureEmbeddedServer.server.start();
-
- TestListenerAdapter tla = new TestListenerAdapter();
- TestNG testng = new TestNG();
- testng.setTestClasses(new Class[] { AdminJerseyResourceIT.class, EntityJerseyResourceIT.class,
- MetadataDiscoveryJerseyResourceIT.class, RexsterGraphJerseyResourceIT.class, TypesJerseyResourceIT.class});
- testng.addListener(tla);
- testng.run();
-
- } finally {
- secureEmbeddedServer.server.stop();
- }
-
- }
-
- protected String getWarPath() {
- return String.format("/target/metadata-webapp-%s",
- System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
- }
-
- protected void setupCredentials() throws Exception {
- Configuration conf = new Configuration(false);
-
- File file = new File(jksPath.toUri().getPath());
- file.delete();
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- // create new aliases
- try {
-
- char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- KEYSTORE_PASSWORD_KEY, storepass);
-
- char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- TRUSTSTORE_PASSWORD_KEY, trustpass);
-
- char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- SERVER_CERT_PASSWORD_KEY, certpass);
-
- // write out so that it can be found in checks
- provider.flush();
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
-}
[09/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java
deleted file mode 100755
index 3e39aac..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/BaseTest.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.junit.Before;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-
-public abstract class BaseTest {
-
- public static final String STRUCT_TYPE_1 = "t1";
- public static final String STRUCT_TYPE_2 = "t2";
- public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
- public static final long TEST_DATE_IN_LONG=1418265358440L;
-
- public static Struct createStruct() throws MetadataException {
- StructType structType = TypeSystem.getInstance().getDataType(
- StructType.class, STRUCT_TYPE_1);
- Struct s = new Struct(structType.getName());
- s.set("a", 1);
- s.set("b", true);
- s.set("c", (byte) 1);
- s.set("d", (short) 2);
- s.set("e", 1);
- s.set("f", 1);
- s.set("g", 1L);
- s.set("h", 1.0f);
- s.set("i", 1.0);
- s.set("j", BigInteger.valueOf(1L));
- s.set("k", new BigDecimal(1));
- s.set("l", new Date(1418265358440L));
- s.set("m", Lists.asList(1, new Integer[]{1}));
- s.set("n", Lists.asList(BigDecimal.valueOf(1.1),
- new BigDecimal[]{BigDecimal.valueOf(1.1)}));
- Map<String, Double> hm = Maps.newHashMap();
- hm.put("a", 1.0);
- hm.put("b", 2.0);
- s.set("o", hm);
- return s;
- }
-
- protected final TypeSystem getTypeSystem() {
- return TypeSystem.getInstance();
- }
-
- @Before
- public void setup() throws Exception {
- TypeSystem ts = TypeSystem.getInstance();
- ts.reset();
-
- StructType structType = ts.defineStructType(STRUCT_TYPE_1,
- true,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
- TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
- TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
- TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
- TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
- TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
- TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
- TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- TypesUtil.createOptionalAttrDef("o",
- ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
- System.out.println("defined structType = " + structType);
-
- StructType recursiveStructType = ts.defineStructType(STRUCT_TYPE_2,
- true,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
- System.out.println("defined recursiveStructType = " + recursiveStructType);
- }
-
- protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs)
- throws MetadataException {
-
- return getTypeSystem().defineTraitTypes(tDefs);
- }
-
- protected Map<String, IDataType> defineClasses(
- HierarchicalTypeDefinition<ClassType>... classDefs) throws MetadataException {
- return getTypeSystem().defineClassTypes(classDefs);
- }
-
- /*
- * Class Hierarchy is:
- * Department(name : String, employees : Array[Person])
- * Person(name : String, department : Department, manager : Manager)
- * Manager(subordinates : Array[Person]) extends Person
- *
- * Persons can have SecurityClearance(level : Int) clearance.
- */
- protected void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
-
- HierarchicalTypeDefinition<ClassType> deptTypeDef = TypesUtil
- .createClassTypeDef("Department",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees",
- String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true,
- "department")
- );
- HierarchicalTypeDefinition<ClassType> personTypeDef = TypesUtil.createClassTypeDef("Person",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department",
- "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager",
- "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- );
- HierarchicalTypeDefinition<ClassType> managerTypeDef =
- TypesUtil.createClassTypeDef("Manager",
- ImmutableList.of("Person"),
- new AttributeDefinition("subordinates",
- String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager")
- );
-
- HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef =
- TypesUtil.createTraitTypeDef(
- "SecurityClearance",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE)
- );
-
- ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(securityClearanceTypeDef),
- ImmutableList.of(deptTypeDef, personTypeDef,
- managerTypeDef));
-
- ImmutableList.of(
- ts.getDataType(HierarchicalType.class, "SecurityClearance"),
- ts.getDataType(ClassType.class, "Department"),
- ts.getDataType(ClassType.class, "Person"),
- ts.getDataType(ClassType.class, "Manager")
- );
- }
-
- protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
- Referenceable hrDept = new Referenceable("Department");
- Referenceable john = new Referenceable("Person");
- Referenceable jane = new Referenceable("Manager", "SecurityClearance");
-
- hrDept.set("name", "hr");
- john.set("name", "John");
- john.set("department", hrDept);
- jane.set("name", "Jane");
- jane.set("department", hrDept);
-
- john.set("manager", jane);
-
- hrDept.set("employees", ImmutableList.of(john, jane));
-
- jane.set("subordinates", ImmutableList.of(john));
-
- jane.getTrait("SecurityClearance").set("level", 1);
-
- ClassType deptType = ts.getDataType(ClassType.class, "Department");
- deptType.convert(hrDept, Multiplicity.REQUIRED);
-
- return hrDept;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ClassTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ClassTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ClassTest.java
deleted file mode 100755
index 0246b61..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ClassTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ClassTest extends BaseTest {
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- @Test
- public void test1() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
-
- defineDeptEmployeeTypes(ts);
- Referenceable hrDept = createDeptEg1(ts);
- ClassType deptType = ts.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
-
-
- Assert.assertEquals(hrDept2.toString(), "{\n" +
- "\tid : (type: Department, id: <unassigned>)\n" +
- "\tname : \thr\n" +
- "\temployees : \t[{\n" +
- "\tid : (type: Person, id: <unassigned>)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : (type: Manager, id: <unassigned>)\n" +
- "}, {\n" +
- "\tid : (type: Manager, id: <unassigned>)\n" +
- "\tsubordinates : \t[{\n" +
- "\tid : (type: Person, id: <unassigned>)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : (type: Manager, id: <unassigned>)\n" +
- "}]\n" +
- "\tname : \tJane\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : <null>\n" +
- "\n" +
- "\tSecurityClearance : \t{\n" +
- "\t\tlevel : \t\t1\n" +
- "\t}}]\n" +
- "}");
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/EnumTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/EnumTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/EnumTest.java
deleted file mode 100755
index a7185d8..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/EnumTest.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-
-public class EnumTest extends BaseTest {
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- void defineEnums(TypeSystem ts) throws MetadataException {
- ts.defineEnumType("HiveObjectType",
- new EnumValue("GLOBAL", 1),
- new EnumValue("DATABASE", 2),
- new EnumValue("TABLE", 3),
- new EnumValue("PARTITION", 4),
- new EnumValue("COLUMN", 5));
-
- ts.defineEnumType("PrincipalType",
- new EnumValue("USER", 1),
- new EnumValue("ROLE", 2),
- new EnumValue("GROUP", 3));
-
- ts.defineEnumType("TxnState",
- new EnumValue("COMMITTED", 1),
- new EnumValue("ABORTED", 2),
- new EnumValue("OPEN", 3));
-
- ts.defineEnumType("LockLevel",
- new EnumValue("DB", 1),
- new EnumValue("TABLE", 2),
- new EnumValue("PARTITION", 3));
-
- }
-
- protected void fillStruct(Struct s) throws MetadataException {
- s.set("a", 1);
- s.set("b", true);
- s.set("c", (byte) 1);
- s.set("d", (short) 2);
- s.set("e", 1);
- s.set("f", 1);
- s.set("g", 1L);
- s.set("h", 1.0f);
- s.set("i", 1.0);
- s.set("j", BigInteger.valueOf(1L));
- s.set("k", new BigDecimal(1));
- s.set("l", new Date(1418265358440L));
- s.set("m", Lists.asList(1, new Integer[]{1}));
- s.set("n",
- Lists.asList(BigDecimal.valueOf(1.1), new BigDecimal[]{BigDecimal.valueOf(1.1)}));
- Map<String, Double> hm = Maps.newHashMap();
- hm.put("a", 1.0);
- hm.put("b", 2.0);
- s.set("o", hm);
- s.set("enum1", "GLOBAL");
- s.set("enum2", 1);
- s.set("enum3", "COMMITTED");
- s.set("enum4", 3);
- }
-
- protected Struct createStructWithEnum(String typeName) throws MetadataException {
- Struct s = new Struct(typeName);
- fillStruct(s);
- return s;
- }
-
- protected Referenceable createInstanceWithEnum(String typeName) throws MetadataException {
- Referenceable r = new Referenceable(typeName);
- fillStruct(r);
- return r;
- }
-
- protected ClassType defineClassTypeWithEnum(TypeSystem ts) throws MetadataException {
- return ts.defineClassType(createClassTypeDef("t4",
- ImmutableList.<String>of(),
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
- createOptionalAttrDef("enum1", ts.getDataType(EnumType.class, "HiveObjectType")),
- createOptionalAttrDef("e", DataTypes.INT_TYPE),
- createOptionalAttrDef("f", DataTypes.INT_TYPE),
- createOptionalAttrDef("g", DataTypes.LONG_TYPE),
- createOptionalAttrDef("enum2", ts.getDataType(EnumType.class, "PrincipalType")),
- createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
- createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
- createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
- createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
- createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
- createOptionalAttrDef("l", DataTypes.DATE_TYPE),
- createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- createOptionalAttrDef("o",
- ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- createOptionalAttrDef("enum4", ts.getDataType(EnumType.class, "LockLevel"))));
- }
-
- @Test
- public void testStruct() throws MetadataException {
- TypeSystem ts = getTypeSystem();
- defineEnums(ts);
- StructType structType = ts.defineStructType("t3",
- true,
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
- createOptionalAttrDef("enum1", ts.getDataType(EnumType.class, "HiveObjectType")),
- createOptionalAttrDef("e", DataTypes.INT_TYPE),
- createOptionalAttrDef("f", DataTypes.INT_TYPE),
- createOptionalAttrDef("g", DataTypes.LONG_TYPE),
- createOptionalAttrDef("enum2", ts.getDataType(EnumType.class, "PrincipalType")),
- createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
- createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
- createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
- createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
- createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
- createOptionalAttrDef("l", DataTypes.DATE_TYPE),
- createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- createOptionalAttrDef("o",
- ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- createOptionalAttrDef("enum4", ts.getDataType(EnumType.class, "LockLevel")));
-
- Struct s = createStructWithEnum("t3");
- ITypedStruct typedS = structType.convert(s, Multiplicity.REQUIRED);
- Assert.assertEquals(typedS.toString(), "{\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t2\n" +
- "\tenum1 : \tGLOBAL\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\tenum2 : \tUSER\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tenum3 : \tCOMMITTED\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.1, 1.1]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "\tenum4 : \tPARTITION\n" +
- "}");
- }
-
- @Test
- public void testClass() throws MetadataException {
- TypeSystem ts = getTypeSystem();
- defineEnums(ts);
- ClassType clsType = defineClassTypeWithEnum(ts);
-
- IReferenceableInstance r = createInstanceWithEnum("t4");
- ITypedReferenceableInstance typedR = clsType.convert(r, Multiplicity.REQUIRED);
- Assert.assertEquals(typedR.toString(), "{\n" +
- "\tid : (type: t4, id: <unassigned>)\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t2\n" +
- "\tenum1 : \tGLOBAL\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\tenum2 : \tUSER\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tenum3 : \tCOMMITTED\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.1, 1.1]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "\tenum4 : \tPARTITION\n" +
- "}");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/StructTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/StructTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/StructTest.java
deleted file mode 100755
index 0db0a67..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/StructTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class StructTest extends BaseTest {
-
- StructType structType;
- StructType recursiveStructType;
-
- @Before
- public void setup() throws Exception {
- super.setup();
- structType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
- recursiveStructType = getTypeSystem()
- .getDataType(StructType.class, STRUCT_TYPE_2);
- }
-
- @Test
- public void test1() throws MetadataException {
- Struct s = createStruct();
- ITypedStruct ts = structType.convert(s, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t2\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.1, 1.1]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "}");
- }
-
- @Test
- public void testRecursive() throws MetadataException {
- Struct s1 = new Struct(recursiveStructType.getName());
- s1.set("a", 1);
- Struct s2 = new Struct(recursiveStructType.getName());
- s2.set("a", 1);
- s2.set("s", s1);
- ITypedStruct ts = recursiveStructType.convert(s2, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\ta : \t1\n" +
- "\ts : \t{\n" +
- "\t\ta : \t\t1\n" +
- "\t\ts : <null>\n" +
- "\n" +
- "\t}\n" +
- "}");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TraitTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TraitTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TraitTest.java
deleted file mode 100755
index e08ed05..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TraitTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-
-public class TraitTest extends BaseTest {
-
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- /*
- * Type Hierarchy is:
- * A(a,b,c,d)
- * B(b) extends A
- * C(c) extends A
- * D(d) extends B,C
- *
- * - There are a total of 11 fields in an instance of D
- * - an attribute that is hidden by a SubType can referenced by prefixing it with the
- * complete Path.
- * For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
- * in B.
- * So it is availabel by the name 'A.B.D.b'
- *
- * - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
- * .b' attribute
- * available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
- * available as 'b'.
- */
- @Test
- public void test1() throws MetadataException {
- HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
- HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
- HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
- HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
-
- defineTraits(A, B, C, D);
-
- TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
-
-// for(String aName : DType.fieldMapping().fields.keySet()) {
-// System.out.println(String.format("nameToQualifiedName.put(\"%s\", \"%s\");", aName, DType.getQualifiedName(aName)));
-// }
-
- Map<String,String> nameToQualifiedName = new HashMap();
- {
- nameToQualifiedName.put("d", "D.d");
- nameToQualifiedName.put("b", "B.b");
- nameToQualifiedName.put("c", "C.c");
- nameToQualifiedName.put("a", "A.a");
- nameToQualifiedName.put("A.B.D.b", "A.B.D.b");
- nameToQualifiedName.put("A.B.D.c", "A.B.D.c");
- nameToQualifiedName.put("A.B.D.d", "A.B.D.d");
- nameToQualifiedName.put("A.C.D.a", "A.C.D.a");
- nameToQualifiedName.put("A.C.D.b", "A.C.D.b");
- nameToQualifiedName.put("A.C.D.c", "A.C.D.c");
- nameToQualifiedName.put("A.C.D.d", "A.C.D.d");
- }
-
- Struct s1 = new Struct("D");
- s1.set("d", 1);
- s1.set("c", 1);
- s1.set("b", true);
- s1.set("a", 1);
- s1.set("A.B.D.b", true);
- s1.set("A.B.D.c", 2);
- s1.set("A.B.D.d", 2);
-
- s1.set("A.C.D.a", 3);
- s1.set("A.C.D.b", false);
- s1.set("A.C.D.c", 3);
- s1.set("A.C.D.d", 3);
-
-
- ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- /*
- * cast to B and set the 'b' attribute on A.
- */
- TraitType BType = (TraitType) getTypeSystem().getDataType(TraitType.class, "B");
- IStruct s2 = DType.castAs(ts, "B");
- s2.set("A.B.b", false);
-
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \tfalse\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- /*
- * cast again to A and set the 'b' attribute on A.
- */
- TraitType AType = (TraitType) getTypeSystem().getDataType(TraitType.class, "A");
- IStruct s3 = BType.castAs(s2, "A");
- s3.set("b", true);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
- }
-
- @Test
- public void testRandomOrder() throws MetadataException {
- HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
- HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
- HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
- HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
-
- defineTraits(B, D, A, C);
-
- TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
-
- Struct s1 = new Struct("D");
- s1.set("d", 1);
- s1.set("c", 1);
- s1.set("b", true);
- s1.set("a", 1);
- s1.set("A.B.D.b", true);
- s1.set("A.B.D.c", 2);
- s1.set("A.B.D.d", 2);
-
- s1.set("A.C.D.a", 3);
- s1.set("A.C.D.b", false);
- s1.set("A.C.D.c", 3);
- s1.set("A.C.D.d", 3);
-
-
- ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeInheritanceTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeInheritanceTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeInheritanceTest.java
deleted file mode 100644
index 447e1be..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeInheritanceTest.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-
-/**
- * Unit tests for type inheritance.
- */
-public class TypeInheritanceTest extends BaseTest {
-
- @BeforeMethod
- public void setup() throws Exception {
- TypeSystem.getInstance().reset();
- super.setup();
- }
-
- /*
- * Type Hierarchy is:
- * A(a)
- * B(b) extends A
- */
- @Test
- public void testSimpleInheritance() throws MetadataException {
- HierarchicalTypeDefinition A = createClassTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE));
-
- HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
-
- defineClasses(A, B);
-
- ClassType BType = getTypeSystem().getDataType(ClassType.class, "B");
-
- Struct s1 = new Struct("B");
- s1.set("b", true);
- s1.set("a", 1);
-
- ITypedInstance ts = BType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\tid : (type: B, id: <unassigned>)\n" +
- "\tb : \ttrue\n" +
- "\ta : \t1\n" +
- "}");
- }
-
- /*
- * Type Hierarchy is:
- * A(a, b)
- * B(b) extends A
- */
- @Test
- public void testSimpleInheritanceWithOverrides() throws MetadataException {
- HierarchicalTypeDefinition A = createClassTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createRequiredAttrDef("b", DataTypes.BOOLEAN_TYPE));
-
- HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
-
- defineClasses(A, B);
-
- ClassType BType = getTypeSystem().getDataType(ClassType.class, "B");
-
- Struct s1 = new Struct("B");
- s1.set("b", true);
- s1.set("a", 1);
- s1.set("A.B.b", false);
-
- ITypedInstance ts = BType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\tid : (type: B, id: <unassigned>)\n" +
- "\tb : \ttrue\n" +
- "\ta : \t1\n" +
- "\tA.B.b : \tfalse\n" +
- "}");
- }
-
- /*
- * Type Hierarchy is:
- * A(a)
- * B(b) extends A
- * C(c) extends B
- * D(d) extends C
- */
- @Test
- public void testMultiLevelInheritance() throws MetadataException {
- HierarchicalTypeDefinition A = createClassTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE));
-
- HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
-
- HierarchicalTypeDefinition C = createClassTypeDef("C", ImmutableList.of("B"),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
-
- HierarchicalTypeDefinition D = createClassTypeDef("D", ImmutableList.of("C"),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
-
- defineClasses(A, B, C, D);
-
- ClassType DType = getTypeSystem().getDataType(ClassType.class, "D");
-
- Struct s1 = new Struct("D");
- s1.set("d", 1);
- s1.set("c", 1);
- s1.set("b", true);
- s1.set("a", 1);
-
- ITypedInstance ts = DType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\tid : (type: D, id: <unassigned>)\n" +
- "\td : \t1\n" +
- "\tc : \t1\n" +
- "\tb : \ttrue\n" +
- "\ta : \t1\n" +
- "}");
- }
-
- /*
- * Type Hierarchy is:
- * A(a,b,c,d)
- * B(b) extends A
- * C(c) extends A
- * D(d) extends B,C
- *
- * - There are a total of 11 fields in an instance of D
- * - an attribute that is hidden by a SubType can referenced by prefixing it with the
- * complete Path.
- * For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
- * in B.
- * So it is availabel by the name 'A.B.D.b'
- *
- * - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
- * .b' attribute
- * available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
- * available as 'b'.
- */
- @Test
- public void testDiamondInheritance() throws MetadataException {
- HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
- HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
- HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.of("A"),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
- HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.of("B", "C"),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
-
- defineTraits(A, B, C, D);
-
- TraitType DType = getTypeSystem().getDataType(TraitType.class, "D");
-
- Struct s1 = new Struct("D");
- s1.set("d", 1);
- s1.set("c", 1);
- s1.set("b", true);
- s1.set("a", 1);
- s1.set("A.B.D.b", true);
- s1.set("A.B.D.c", 2);
- s1.set("A.B.D.d", 2);
-
- s1.set("A.C.D.a", 3);
- s1.set("A.C.D.b", false);
- s1.set("A.C.D.c", 3);
- s1.set("A.C.D.d", 3);
-
-
- ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- /*
- * cast to B and set the 'b' attribute on A.
- */
- TraitType BType = getTypeSystem().getDataType(TraitType.class, "B");
- IStruct s2 = DType.castAs(ts, "B");
- s2.set("A.B.b", false);
-
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \tfalse\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- /*
- * cast again to A and set the 'b' attribute on A.
- */
- IStruct s3 = BType.castAs(s2, "A");
- s3.set("b", true);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeSystemTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeSystemTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeSystemTest.java
deleted file mode 100755
index 3b9a4a5..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/TypeSystemTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-import scala.actors.threadpool.Arrays;
-
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createStructTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-
-public class TypeSystemTest extends BaseTest {
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setup();
- }
-
- @AfterMethod
- public void tearDown() throws Exception {
- getTypeSystem().reset();
- }
-
- @Test
- public void testGetTypeNames() throws Exception {
- getTypeSystem().defineEnumType("enum_test",
- new EnumValue("0", 0),
- new EnumValue("1", 1),
- new EnumValue("2", 2),
- new EnumValue("3", 3));
- Assert.assertTrue(getTypeSystem().getTypeNames().contains("enum_test"));
- }
-
- @Test
- public void testIsRegistered() throws Exception {
- getTypeSystem().defineEnumType("enum_test",
- new EnumValue("0", 0),
- new EnumValue("1", 1),
- new EnumValue("2", 2),
- new EnumValue("3", 3));
- Assert.assertTrue(getTypeSystem().isRegistered("enum_test"));
- }
-
- @Test
- public void testGetTraitsNames() throws Exception {
- HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
- TypesUtil.createTraitTypeDef("Classification",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef("PII", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> phiTrait =
- TypesUtil.createTraitTypeDef("PHI", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> pciTrait =
- TypesUtil.createTraitTypeDef("PCI", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> soxTrait =
- TypesUtil.createTraitTypeDef("SOX", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> secTrait =
- TypesUtil.createTraitTypeDef("SEC", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> financeTrait =
- TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
-
- getTypeSystem().defineTypes(
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
- soxTrait, secTrait, financeTrait),
- ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
-
- final ImmutableList<String> traitsNames = getTypeSystem().getTypeNamesByCategory(DataTypes.TypeCategory.TRAIT);
- Assert.assertEquals(traitsNames.size(), 7);
- List traits = Arrays.asList(new String[]{
- "Classification",
- "PII",
- "PHI",
- "PCI",
- "SOX",
- "SEC",
- "Finance",
- });
-
- Assert.assertFalse(Collections.disjoint(traitsNames, traits));
- }
-
- private String random() {
- return RandomStringUtils.random(10);
- }
-
- @Test
- public void testUTFNames() throws Exception {
- TypeSystem ts = getTypeSystem();
-
- String enumType = random();
- EnumTypeDefinition orgLevelEnum =
- new EnumTypeDefinition(enumType, new EnumValue(random(), 1), new EnumValue(random(), 2));
- ts.defineEnumType(orgLevelEnum);
-
- String structName = random();
- String attrType = random();
- StructTypeDefinition structType = createStructTypeDef(structName,
- createRequiredAttrDef(attrType, DataTypes.STRING_TYPE));
-
- String className = random();
- HierarchicalTypeDefinition<ClassType> classType =
- createClassTypeDef(className, ImmutableList.<String>of(),
- createRequiredAttrDef(attrType, DataTypes.STRING_TYPE));
-
- String traitName = random();
- HierarchicalTypeDefinition<TraitType> traitType = createTraitTypeDef(traitName,
- ImmutableList.<String>of(), createRequiredAttrDef(attrType, DataTypes.INT_TYPE));
-
- ts.defineTypes(ImmutableList.of(structType), ImmutableList.of(traitType), ImmutableList.of(classType));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ValidationTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ValidationTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ValidationTest.java
deleted file mode 100644
index 24a3950..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/types/ValidationTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-public class ValidationTest {
- @DataProvider(name = "attributeData")
- private Object[][] createAttributeData() {
- return new String[][]{
- {null, "type"}, {"", "type"}, {"name", null}, {"name", ""}};
- }
-
- @Test (dataProvider = "attributeData", expectedExceptions = {IllegalArgumentException.class})
- public void testAttributes(String name, String type) {
- TypesUtil.createRequiredAttrDef(name, type);
- }
-
- @DataProvider(name = "enumValueData")
- private Object[][] createEnumValueData() {
- return new String[][]{{null}, {""}};
- }
-
- @Test (dataProvider = "enumValueData", expectedExceptions = {IllegalArgumentException.class})
- public void testEnumValue(String name) {
- new EnumValue(name, 1);
- }
-
- @DataProvider(name = "enumTypeData")
- private Object[][] createEnumTypeData() {
- EnumValue value = new EnumValue("name", 1);
- return new Object[][]{{null, value}, {"", value}, {"name"}};
- }
-
- @Test (dataProvider = "enumTypeData", expectedExceptions = {IllegalArgumentException.class})
- public void testEnumType(String name, EnumValue... values) {
- new EnumTypeDefinition(name, values);
- }
-
- @DataProvider(name = "structTypeData")
- private Object[][] createStructTypeData() {
- AttributeDefinition value = TypesUtil.createRequiredAttrDef("name", "type");
- return new Object[][]{{null, value}, {"", value}, {"name"}};
- }
-
- @Test (dataProvider = "structTypeData", expectedExceptions = {IllegalArgumentException.class})
- public void testStructType(String name, AttributeDefinition... values) {
- new StructTypeDefinition(name, values);
- }
-
- @DataProvider(name = "classTypeData")
- private Object[][] createClassTypeData() {
- return new Object[][]{{null}, {""}};
- }
-
- @Test (dataProvider = "classTypeData", expectedExceptions = {IllegalArgumentException.class})
- public void testClassType(String name) {
- AttributeDefinition value = TypesUtil.createRequiredAttrDef("name", "type");;
- TypesUtil.createClassTypeDef(name, ImmutableList.of("super"), value);
- }
-
- @Test (dataProvider = "classTypeData", expectedExceptions = {IllegalArgumentException.class})
- public void testTraitType(String name) {
- AttributeDefinition value = TypesUtil.createRequiredAttrDef("name", "type");;
- TypesUtil.createTraitTypeDef(name, ImmutableList.of("super"), value);
- }
-
- @Test
- public void testValidTypes() {
- AttributeDefinition attribute = TypesUtil.createRequiredAttrDef("name", "type");
-
- //class with no attributes
- TypesUtil.createClassTypeDef("name", ImmutableList.of("super"));
-
- //class with no super types
- TypesUtil.createClassTypeDef("name", ImmutableList.<String>of(), attribute);
-
- //trait with no attributes
- TypesUtil.createTraitTypeDef("name", ImmutableList.of("super"));
-
- //trait with no super types
- TypesUtil.createTraitTypeDef("name", ImmutableList.<String>of(), attribute);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/BuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/BuilderTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/BuilderTest.scala
new file mode 100644
index 0000000..b6e3318
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/BuilderTest.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.builders
+
+import org.apache.atlas.typesystem.TypesDef
+import org.apache.atlas.typesystem.types.TypeSystem
+import org.scalatest.{BeforeAndAfter, FunSuite}
+
+abstract class BuilderTest extends FunSuite with BeforeAndAfter {
+
+ var tDef : TypesDef = null
+
+ before {
+ TypeSystem.getInstance().reset()
+
+ val b = new TypesBuilder
+ import b._
+
+ tDef = types {
+
+ _trait("Dimension") {}
+ _trait("PII") {}
+ _trait("Metric") {}
+ _trait("ETL") {}
+ _trait("JdbcAccess") {}
+
+ _class("DB") {
+ "name" ~ (string, required, indexed, unique)
+ "owner" ~ (string)
+ "createTime" ~ (int)
+ }
+
+ _class("StorageDesc") {
+ "inputFormat" ~ (string, required)
+ "outputFormat" ~ (string, required)
+ }
+
+ _class("Column") {
+ "name" ~ (string, required)
+ "dataType" ~ (string, required)
+ "sd" ~ ("StorageDesc", required)
+ }
+
+ _class("Table", List()) {
+ "name" ~ (string, required, indexed)
+ "db" ~ ("DB", required)
+ "sd" ~ ("StorageDesc", required)
+ }
+
+ _class("LoadProcess") {
+ "name" ~ (string, required)
+ "inputTables" ~ (array("Table"), collection)
+ "outputTable" ~ ("Table", required)
+
+ }
+
+ _class("View") {
+ "name" ~ (string, required)
+ "inputTables" ~ (array("Table"), collection)
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/InstanceBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/InstanceBuilderTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/InstanceBuilderTest.scala
new file mode 100644
index 0000000..83fb817
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/InstanceBuilderTest.scala
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.builders
+
+import org.apache.atlas.typesystem.types.{ClassType, Multiplicity, TypeSystem}
+
+
+class InstanceBuilderTest extends BuilderTest {
+
+ test("test1") {
+ TypeSystem.getInstance().defineTypes(tDef)
+
+ val b = new InstanceBuilder
+ import b._
+
+ val instances = b create {
+
+ val salesDB = instance("DB") { // use instance to create Referenceables. use closure to
+ // set attributes of instance
+ 'name ~ "Sales" // use '~' to set attributes. Use a Symbol (names starting with ') for
+ // attribute names.
+ 'owner ~ "John ETL"
+ 'createTime ~ 1000
+ }
+
+ val salesFact = instance("Table") {
+ 'name ~ "sales_fact"
+ 'db ~ salesDB
+ val sd = instance("StorageDesc") { // any valid scala allowed in closure.
+ 'inputFormat ~ "TextIputFormat"
+ 'outputFormat ~ "TextOutputFormat"
+ }
+ 'sd ~ sd // use ~ to set references, collections and maps.
+ val columns = Seq(
+ instance("Column") {
+ 'name ~ "time_id"
+ 'dataType ~ "int"
+ 'sd ~ sd
+ },
+ instance("Column") {
+ 'name ~ "product_id"
+ 'dataType ~ "int"
+ 'sd ~ sd
+ },
+ instance("Column") {
+ 'name ~ "customer_id"
+ 'dataType ~ "int"
+ 'sd ~ sd
+ },
+ instance("Column", "Metric") {
+ 'name ~ "sales"
+ 'dataType ~ "int"
+ 'sd ~ sd
+ 'Metric("x") ~ 1 // use 'TraitName("attrName") to set values on traits.
+ }
+ )
+
+ 'columns ~ columns
+
+ }
+
+ salesFact.sd.inputFormat ~ "TextInputFormat" // use dot navigation to alter attributes in the object graph.
+ // here I am fixing the typo in "TextInputFormat"
+ // dot navigation also works for arrays.
+ // here I am fixing column(3). Metric trait has no attributes.
+ val c = salesFact.columns
+ c(3) = instance("Column", "Metric") {
+ 'name ~ "sales"
+ 'dataType ~ "int"
+ 'sd ~ salesFact.sd
+ }
+
+ }
+
+ val ts = TypeSystem.getInstance()
+
+ import scala.collection.JavaConversions._
+ val typedInstances = instances.map { i =>
+ val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
+ iTyp.convert(i, Multiplicity.REQUIRED)
+ }
+
+ typedInstances.foreach { i =>
+ println(i)
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/MultiplicityTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/MultiplicityTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/MultiplicityTest.scala
new file mode 100644
index 0000000..eea904a
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/MultiplicityTest.scala
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.builders
+
+import org.apache.atlas.MetadataException
+import org.apache.atlas.typesystem.types.{ClassType, Multiplicity, TypeSystem}
+import org.scalatest.{BeforeAndAfterAll, FunSuite}
+
+
+class MultiplicityTest extends FunSuite with BeforeAndAfterAll {
+
+ override def beforeAll() = {
+ TypeSystem.getInstance().reset()
+
+ val b = new TypesBuilder
+ import b._
+
+ val tDef = types {
+
+ _trait("Dimension") {}
+ _trait("PII") {}
+ _trait("Metric") {}
+ _trait("ETL") {}
+ _trait("JdbcAccess") {}
+
+ _class("DB") {
+ "name" ~ (string, required, indexed, unique)
+ "owner" ~ (string)
+ "createTime" ~ (int)
+ }
+
+ _class("StorageDesc") {
+ "inputFormat" ~ (string, required)
+ "outputFormat" ~ (string, required)
+ }
+
+ _class("Column") {
+ "name" ~ (string, required)
+ "dataType" ~ (string, required)
+ "sd" ~ ("StorageDesc", required)
+ }
+
+ _class("Table", List()) {
+ "name" ~ (string, required, indexed)
+ "db" ~ ("DB", required)
+ "sd" ~ ("StorageDesc", required)
+ }
+
+ _class("LoadProcess") {
+ "name" ~ (string, required)
+ "inputTables" ~ (array("Table"), collection)
+ "outputTable" ~ ("Table", required)
+
+ }
+
+ _class("View") {
+ "name" ~ (string, required)
+ "inputTables" ~ (array("Table"), collection)
+ }
+
+ _class("AT") {
+ "name" ~ (string, required)
+ "stringSet" ~ (array("string"), multiplicty(0, Int.MaxValue, true))
+ }
+ }
+
+ TypeSystem.getInstance().defineTypes(tDef)
+ }
+
+ test("test1") {
+
+ val b = new InstanceBuilder
+ import b._
+
+ val instances = b create {
+ val a = instance("AT") { // use instance to create Referenceables. use closure to
+ // set attributes of instance
+ 'name ~ "A1" // use '~' to set attributes. Use a Symbol (names starting with ') for
+ 'stringSet ~ Seq("a", "a")
+ }
+ }
+
+ val ts = TypeSystem.getInstance()
+ import scala.collection.JavaConversions._
+ val typedInstances = instances.map { i =>
+ val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
+ iTyp.convert(i, Multiplicity.REQUIRED)
+ }
+
+ typedInstances.foreach { i =>
+ println(i)
+ }
+ }
+
+ test("WrongMultiplicity") {
+ val b = new TypesBuilder
+ import b._
+ val tDef = types {
+ _class("Wrong") {
+ "name" ~ (string, required)
+ "stringSet" ~ (string, multiplicty(0, Int.MaxValue, true))
+ }
+ }
+ val me = intercept[MetadataException] {
+ TypeSystem.getInstance().defineTypes(tDef)
+ }
+ assert("A multiplicty of more than one requires a collection type for attribute 'stringSet'" == me.getMessage)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/TypesBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/TypesBuilderTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/TypesBuilderTest.scala
new file mode 100644
index 0000000..4eb8f3c
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/builders/TypesBuilderTest.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.builders
+
+import org.apache.atlas.typesystem.json.TypesSerialization
+import org.apache.atlas.typesystem.types.TypeSystem
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class TypesBuilderTest extends BuilderTest {
+
+
+ test("test1") {
+ TypeSystem.getInstance().defineTypes(tDef)
+
+ println(TypesSerialization.toJson(TypeSystem.getInstance(), x => true))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
new file mode 100755
index 0000000..5556d77
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json
+
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.typesystem.persistence.{Id, ReferenceableInstance, StructInstance}
+import org.apache.atlas.typesystem.types._
+import org.apache.atlas.typesystem.types.utils.TypesUtil
+import org.apache.atlas.typesystem.{ITypedReferenceableInstance, ITypedStruct, Referenceable, Struct}
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.{write => swrite, _}
+import org.json4s.{NoTypeHints, _}
+import org.junit.{Assert, Before, Test}
+
+class SerializationTest extends BaseTest {
+
+ private[atlas] var structType: StructType = null
+ private[atlas] var recursiveStructType: StructType = null
+
+ @Before
+ override def setup {
+ super.setup
+ structType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_1).asInstanceOf[StructType]
+ recursiveStructType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_2).asInstanceOf[StructType]
+ }
+
+ @Test def test1 {
+ val s: Struct = BaseTest.createStruct()
+ val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
+
+ Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
+
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new BigDecimalSerializer + new BigIntegerSerializer
+
+ //Json representation
+ val ser = swrite(ts)
+ Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"" + BaseTest.TEST_DATE + "\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
+
+ // Typed Struct read back
+ val ts1 = read[StructInstance](ser)
+ Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
+ }
+
+ @Test def test2 {
+ val s: Struct = BaseTest.createStruct()
+ val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
+
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new BigDecimalSerializer + new BigIntegerSerializer
+
+ val ts1 = read[StructInstance](
+ """
+ {"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
+ "c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}""")
+ // Typed Struct read from string
+ Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
+ }
+
+ @Test def testTrait {
+ val A: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef("A", null,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
+ val B: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
+ "B", ImmutableList.of[String]("A"),
+ TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE))
+ val C: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
+ "C", ImmutableList.of[String]("A"),
+ TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE))
+ val D: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
+ "D", ImmutableList.of[String]("B", "C"),
+ TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
+
+ defineTraits(A, B, C, D)
+
+ val DType: TraitType = getTypeSystem.getDataType(classOf[TraitType], "D").asInstanceOf[TraitType]
+ val s1: Struct = new Struct("D")
+ s1.set("d", 1)
+ s1.set("c", 1)
+ s1.set("b", true)
+ s1.set("a", 1)
+ s1.set("A.B.D.b", true)
+ s1.set("A.B.D.c", 2)
+ s1.set("A.B.D.d", 2)
+ s1.set("A.C.D.a", 3)
+ s1.set("A.C.D.b", false)
+ s1.set("A.C.D.c", 3)
+ s1.set("A.C.D.d", 3)
+
+ val s: Struct = BaseTest.createStruct()
+ val ts: ITypedStruct = DType.convert(s1, Multiplicity.REQUIRED)
+
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new BigDecimalSerializer + new BigIntegerSerializer
+
+ // Typed Struct :
+ Assert.assertEquals(ts.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
+
+ // Json representation :
+ val ser = swrite(ts)
+ Assert.assertEquals(ser, "{\"$typeName$\":\"D\",\"A.C.D.d\":3,\"A.B.D.c\":2,\"b\":true,\"A.C.D.c\":3,\"d\":1,\"A.B.D.b\":true,\"a\":1,\"A.C.D.b\":false,\"A.B.D.d\":2,\"c\":1,\"A.C.D.a\":3}")
+
+ val ts1 = read[StructInstance](
+ """
+ {"$typeName$":"D","A.C.D.d":3,"A.B.D.c":2,"b":true,"A.C.D.c":3,"d":1,
+ "A.B.D.b":true,"a":1,"A.C.D.b":false,"A.B.D.d":2,"c":1,"A.C.D.a":3}""")
+ // Typed Struct read from string:
+ Assert.assertEquals(ts1.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
+ }
+
+ def defineHRTypes(ts: TypeSystem) : Unit = {
+ val deptTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
+ "Department",
+ ImmutableList.of[String],
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, true, "department"))
+ val personTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
+ "Person", ImmutableList.of[String],
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"))
+ val managerTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
+ "Manager", ImmutableList.of[String]("Person"),
+ new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, false, "manager"))
+ val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] =
+ TypesUtil.createTraitTypeDef("SecurityClearance", ImmutableList.of[String],
+ TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE))
+
+ ts.defineTypes(ImmutableList.of[StructTypeDefinition],
+ ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
+ ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef)
+ )
+
+ }
+
+ def defineHRDept() : Referenceable = {
+ val hrDept: Referenceable = new Referenceable("Department")
+ val john: Referenceable = new Referenceable("Person")
+ val jane: Referenceable = new Referenceable("Manager", "SecurityClearance")
+ hrDept.set("name", "hr")
+ john.set("name", "John")
+ john.set("department", hrDept.getId)
+ jane.set("name", "Jane")
+ jane.set("department", hrDept.getId)
+ john.set("manager", jane.getId)
+ hrDept.set("employees", ImmutableList.of[Referenceable](john, jane))
+ jane.set("subordinates", ImmutableList.of[Id](john.getId))
+ jane.getTrait("SecurityClearance").set("level", 1)
+ hrDept
+ }
+
+ @Test def testClass {
+
+ val ts: TypeSystem = getTypeSystem
+ defineHRTypes(ts)
+ val hrDept: Referenceable = defineHRDept()
+
+ val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
+ val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
+
+ println(s"HR Dept Object Graph:\n${hrDept2}\n")
+
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ val ser = swrite(hrDept2)
+ println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
+
+ println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
+ }
+
+ @Test def testReference {
+
+ val ts: TypeSystem = getTypeSystem
+ defineHRTypes(ts)
+ val hrDept: Referenceable = defineHRDept()
+
+
+ val jsonStr = InstanceSerialization.toJson(hrDept)
+ val hrDept2 = InstanceSerialization.fromJsonReferenceable(jsonStr)
+
+ val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
+ val hrDept3: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
+
+ println(s"HR Dept Object Graph:\n${hrDept3}\n")
+
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ val ser = swrite(hrDept3)
+ println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
+
+ println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
+ }
+
+ @Test def testReference2 {
+
+ val ts: TypeSystem = getTypeSystem
+ defineHRTypes(ts)
+ val hrDept: Referenceable = defineHRDept()
+
+ val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
+ val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
+
+ val jsonStr = InstanceSerialization.toJson(hrDept2)
+ val hrDept3 = InstanceSerialization.fromJsonReferenceable(jsonStr)
+
+ val hrDept4: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
+
+ println(s"HR Dept Object Graph:\n${hrDept4}\n")
+
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ val ser = swrite(hrDept4)
+ println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
+
+ println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
+
+ }
+
+}
\ No newline at end of file
[12/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructType.java
deleted file mode 100755
index d24ab6b..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructType.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-public class StructType extends AbstractDataType<IStruct>
- implements IConstructableType<IStruct, ITypedStruct> {
-
- public final TypeSystem typeSystem;
- public final String name;
- public final FieldMapping fieldMapping;
- public final Map<AttributeInfo, List<String>> infoToNameMap;
- public final int numFields;
- private final TypedStructHandler handler;
-
- /**
- * Used when creating a StructType, to support recursive Structs.
- */
- protected StructType(TypeSystem typeSystem, String name, int numFields) {
- this.typeSystem = typeSystem;
- this.name = name;
- this.fieldMapping = null;
- infoToNameMap = null;
- this.numFields = numFields;
- this.handler = null;
- }
-
- protected StructType(TypeSystem typeSystem, String name,
- ImmutableList<String> superTypes, AttributeInfo... fields)
- throws MetadataException {
- this.typeSystem = typeSystem;
- this.name = name;
- this.fieldMapping = constructFieldMapping(superTypes,
- fields);
- infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping);
- this.numFields = this.fieldMapping.fields.size();
- this.handler = new TypedStructHandler(this);
- }
-
- public FieldMapping fieldMapping() {
- return fieldMapping;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- protected FieldMapping constructFieldMapping(ImmutableList<String> superTypes,
- AttributeInfo... fields)
- throws MetadataException {
-
- Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
- Map<String, Integer> fieldPos = new HashMap<String, Integer>();
- Map<String, Integer> fieldNullPos = new HashMap<String, Integer>();
- int numBools = 0;
- int numBytes = 0;
- int numShorts = 0;
- int numInts = 0;
- int numLongs = 0;
- int numFloats = 0;
- int numDoubles = 0;
- int numBigInts = 0;
- int numBigDecimals = 0;
- int numDates = 0;
- int numStrings = 0;
- int numArrays = 0;
- int numMaps = 0;
- int numStructs = 0;
- int numReferenceables = 0;
-
- for (AttributeInfo i : fields) {
- if (fieldsMap.containsKey(i.name)) {
- throw new MetadataException(
- String.format(
- "Struct defintion cannot contain multiple fields with the same " +
- "name %s",
- i.name));
- }
- fieldsMap.put(i.name, i);
- fieldNullPos.put(i.name, fieldNullPos.size());
- if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
- fieldPos.put(i.name, numBools);
- numBools++;
- } else if (i.dataType() == DataTypes.BYTE_TYPE) {
- fieldPos.put(i.name, numBytes);
- numBytes++;
- } else if (i.dataType() == DataTypes.SHORT_TYPE) {
- fieldPos.put(i.name, numShorts);
- numShorts++;
- } else if (i.dataType() == DataTypes.INT_TYPE) {
- fieldPos.put(i.name, numInts);
- numInts++;
- } else if (i.dataType() == DataTypes.LONG_TYPE) {
- fieldPos.put(i.name, numLongs);
- numLongs++;
- } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
- fieldPos.put(i.name, numFloats);
- numFloats++;
- } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
- fieldPos.put(i.name, numDoubles);
- numDoubles++;
- } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
- fieldPos.put(i.name, numBigInts);
- numBigInts++;
- } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- fieldPos.put(i.name, numBigDecimals);
- numBigDecimals++;
- } else if (i.dataType() == DataTypes.DATE_TYPE) {
- fieldPos.put(i.name, numDates);
- numDates++;
- } else if (i.dataType() == DataTypes.STRING_TYPE) {
- fieldPos.put(i.name, numStrings);
- numStrings++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
- fieldPos.put(i.name, numInts);
- numInts++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- fieldPos.put(i.name, numArrays);
- numArrays++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
- fieldPos.put(i.name, numMaps);
- numMaps++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
- i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
- fieldPos.put(i.name, numStructs);
- numStructs++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- fieldPos.put(i.name, numReferenceables);
- numReferenceables++;
- } else {
- throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
- }
- }
-
- return new FieldMapping(fieldsMap,
- fieldPos,
- fieldNullPos,
- numBools,
- numBytes,
- numShorts,
- numInts,
- numLongs,
- numFloats,
- numDoubles,
- numBigInts,
- numBigDecimals,
- numDates,
- numStrings,
- numArrays,
- numMaps,
- numStructs,
- numReferenceables);
- }
-
-
- @Override
- public DataTypes.TypeCategory getTypeCategory() {
- return DataTypes.TypeCategory.STRUCT;
- }
-
- @Override
- public ITypedStruct convert(Object val, Multiplicity m) throws MetadataException {
- return handler.convert(val, m);
- }
-
- public ITypedStruct createInstance() {
- return handler.createInstance();
- }
-
- @Override
- public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
- handler.output(s, buf, prefix);
- }
-
- public List<String> getNames(AttributeInfo info) {
- return infoToNameMap.get(info);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructTypeDefinition.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructTypeDefinition.java
deleted file mode 100755
index 251de4f..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/StructTypeDefinition.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.ParamChecker;
-
-import java.util.Arrays;
-
-public class StructTypeDefinition {
-
- public final String typeName;
- public final AttributeDefinition[] attributeDefinitions;
-
- protected StructTypeDefinition(String typeName, boolean validate, AttributeDefinition... attributeDefinitions) {
- this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
- if (attributeDefinitions != null && attributeDefinitions.length != 0) {
- ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
- }
- this.attributeDefinitions = attributeDefinitions;
- }
-
- public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) {
- this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
- this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- StructTypeDefinition that = (StructTypeDefinition) o;
-
- if (!Arrays.equals(attributeDefinitions, that.attributeDefinitions)) return false;
- if (!typeName.equals(that.typeName)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = typeName.hashCode();
- result = 31 * result + Arrays.hashCode(attributeDefinitions);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TraitType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TraitType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TraitType.java
deleted file mode 100755
index 87181ed..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TraitType.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-
-import java.util.List;
-import java.util.Map;
-
-public class TraitType extends HierarchicalType<TraitType, IStruct>
- implements IConstructableType<IStruct, ITypedStruct> {
-
- public final Map<AttributeInfo, List<String>> infoToNameMap;
- private final TypedStructHandler handler;
-
- /**
- * Used when creating a TraitType, to support recursive Structs.
- */
- TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits,
- int numFields) {
- super(typeSystem, TraitType.class, name, superTraits, numFields);
- handler = null;
- infoToNameMap = null;
- }
-
- TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits,
- AttributeInfo... fields)
- throws MetadataException {
- super(typeSystem, TraitType.class, name, superTraits, fields);
- handler = new TypedStructHandler(this);
- infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
- }
-
- @Override
- public DataTypes.TypeCategory getTypeCategory() {
- return DataTypes.TypeCategory.TRAIT;
- }
-
- @Override
- public ITypedStruct convert(Object val, Multiplicity m) throws MetadataException {
- return handler.convert(val, m);
- }
-
- public ITypedStruct createInstance() {
- return handler.createInstance();
- }
-
- @Override
- public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
- handler.output(s, buf, prefix);
- }
-
- @Override
- public List<String> getNames(AttributeInfo info) {
- return infoToNameMap.get(info);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
deleted file mode 100755
index f44b0bb..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
+++ /dev/null
@@ -1,674 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Multimap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.classification.InterfaceAudience;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-
-import javax.inject.Singleton;
-import java.lang.reflect.Constructor;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-@Singleton
-@InterfaceAudience.Private
-public class TypeSystem {
- private static final TypeSystem INSTANCE = new TypeSystem();
- private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() {
- @Override
- public SimpleDateFormat initialValue() {
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- return dateFormat;
- }
- };
-
- private Map<String, IDataType> types;
- private IdType idType;
-
- /**
- * An in-memory copy of type categories vs types for convenience.
- */
- private Multimap<DataTypes.TypeCategory, String> typeCategoriesToTypeNamesMap;
-
- private ImmutableList<String> coreTypes;
-
- public TypeSystem() {
- initialize();
- }
-
- public static TypeSystem getInstance() {
- return INSTANCE;
- }
-
- /**
- * This is only used for testing purposes. Not intended for public use.
- */
- @InterfaceAudience.Private
- public void reset() {
- initialize();
- }
-
- private void initialize() {
- types = new ConcurrentHashMap<>();
- typeCategoriesToTypeNamesMap = ArrayListMultimap.create(DataTypes.TypeCategory.values().length, 10);
-
- registerPrimitiveTypes();
- registerCoreTypes();
- coreTypes = ImmutableList.copyOf(types.keySet());
- }
-
- public ImmutableList<String> getCoreTypes() {
- return coreTypes;
- }
-
- public ImmutableList<String> getTypeNames() {
- return ImmutableList.copyOf(types.keySet());
- }
-
- public ImmutableList<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) {
- return ImmutableList.copyOf(typeCategoriesToTypeNamesMap.get(typeCategory));
- }
-
- private void registerPrimitiveTypes() {
- types.put(DataTypes.BOOLEAN_TYPE.getName(), DataTypes.BOOLEAN_TYPE);
- types.put(DataTypes.BYTE_TYPE.getName(), DataTypes.BYTE_TYPE);
- types.put(DataTypes.SHORT_TYPE.getName(), DataTypes.SHORT_TYPE);
- types.put(DataTypes.INT_TYPE.getName(), DataTypes.INT_TYPE);
- types.put(DataTypes.LONG_TYPE.getName(), DataTypes.LONG_TYPE);
- types.put(DataTypes.FLOAT_TYPE.getName(), DataTypes.FLOAT_TYPE);
- types.put(DataTypes.DOUBLE_TYPE.getName(), DataTypes.DOUBLE_TYPE);
- types.put(DataTypes.BIGINTEGER_TYPE.getName(), DataTypes.BIGINTEGER_TYPE);
- types.put(DataTypes.BIGDECIMAL_TYPE.getName(), DataTypes.BIGDECIMAL_TYPE);
- types.put(DataTypes.DATE_TYPE.getName(), DataTypes.DATE_TYPE);
- types.put(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE);
-
- typeCategoriesToTypeNamesMap.putAll(DataTypes.TypeCategory.PRIMITIVE, types.keySet());
- }
-
-
- /*
- * The only core OOB type we will define is the Struct to represent the Identity of an Instance.
- */
- private void registerCoreTypes() {
- idType = new IdType();
- }
-
- public IdType getIdType() {
- return idType;
- }
-
- public boolean isRegistered(String typeName) {
- return types.containsKey(typeName);
- }
-
- public <T> T getDataType(Class<T> cls, String name) throws MetadataException {
- if (types.containsKey(name)) {
- try {
- return cls.cast(types.get(name));
- } catch(ClassCastException cce) {
- throw new MetadataException(cce);
- }
- }
-
- /*
- * is this an Array Type?
- */
- String arrElemType = TypeUtils.parseAsArrayType(name);
- if (arrElemType != null) {
- IDataType dT = defineArrayType(getDataType(IDataType.class, arrElemType));
- return cls.cast(dT);
- }
-
- /*
- * is this a Map Type?
- */
- String[] mapType = TypeUtils.parseAsMapType(name);
- if (mapType != null) {
- IDataType dT = defineMapType(getDataType(IDataType.class, mapType[0]),
- getDataType(IDataType.class, mapType[1]));
- return cls.cast(dT);
- }
-
- throw new MetadataException(String.format("Unknown datatype: %s", name));
- }
-
- public StructType defineStructType(String name,
- boolean errorIfExists,
- AttributeDefinition... attrDefs) throws MetadataException {
- StructTypeDefinition structDef = new StructTypeDefinition(name, attrDefs);
- defineTypes(ImmutableList.of(structDef),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
-
- return getDataType(StructType.class, structDef.typeName);
- }
-
- /**
- * construct a temporary StructType for a Query Result. This is not registered in the
- * typeSystem.
- * The attributes in the typeDefinition can only reference permanent types.
- * @param name struct type name
- * @param attrDefs struct type definition
- * @return temporary struct type
- * @throws MetadataException
- */
- public StructType defineQueryResultType(String name,
- Map<String, IDataType> tempTypes,
- AttributeDefinition... attrDefs)
- throws MetadataException {
-
- AttributeInfo[] infos = new AttributeInfo[attrDefs.length];
- for (int i = 0; i < attrDefs.length; i++) {
- infos[i] = new AttributeInfo(this, attrDefs[i], tempTypes);
- }
-
- return new StructType(this, name, null, infos);
- }
-
- public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef)
- throws MetadataException {
-
- defineTypes(ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(traitDef),
- ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
-
- return getDataType(TraitType.class, traitDef.typeName);
- }
-
- public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef)
- throws MetadataException {
-
- defineTypes(ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.of(classDef));
-
- return getDataType(ClassType.class, classDef.typeName);
- }
-
- public Map<String, IDataType> defineTraitTypes(
- HierarchicalTypeDefinition<TraitType>... traitDefs) throws MetadataException {
- TransientTypeSystem transientTypes = new TransientTypeSystem(
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.copyOf(traitDefs),
- ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
- return transientTypes.defineTypes();
- }
-
- public Map<String, IDataType> defineClassTypes(
- HierarchicalTypeDefinition<ClassType>... classDefs) throws MetadataException {
- TransientTypeSystem transientTypes = new TransientTypeSystem(
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.copyOf(classDefs));
- return transientTypes.defineTypes();
- }
-
- public Map<String, IDataType> defineTypes(TypesDef typesDef)
- throws MetadataException {
- Map<String, IDataType> typesAdded = new HashMap<>();
- for (EnumTypeDefinition enumDef : typesDef.enumTypesAsJavaList()) {
- typesAdded.put(enumDef.name, defineEnumType(enumDef));
- }
-
- ImmutableList<StructTypeDefinition> structDefs = ImmutableList
- .copyOf(typesDef.structTypesAsJavaList());
- ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs =
- ImmutableList.copyOf(typesDef.traitTypesAsJavaList());
- ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs =
- ImmutableList.copyOf(typesDef.classTypesAsJavaList());
-
- typesAdded.putAll(defineTypes(structDefs, traitDefs, classDefs));
- return typesAdded;
- }
-
- public Map<String, IDataType> defineTypes(ImmutableList<StructTypeDefinition> structDefs,
- ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
- ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs)
- throws MetadataException {
- TransientTypeSystem transientTypes = new TransientTypeSystem(structDefs,
- traitDefs,
- classDefs);
- Map<String, IDataType> definedTypes = transientTypes.defineTypes();
-// LOG.debug("Defined new types " + Arrays.toString(definedTypes.keySet().toArray(new String[definedTypes.size()])));
- return definedTypes;
- }
-
- public DataTypes.ArrayType defineArrayType(IDataType elemType) throws MetadataException {
- assert elemType != null;
- DataTypes.ArrayType dT = new DataTypes.ArrayType(elemType);
- types.put(dT.getName(), dT);
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ARRAY, dT.getName());
- return dT;
- }
-
- public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType)
- throws MetadataException {
- assert keyType != null;
- assert valueType != null;
- DataTypes.MapType dT = new DataTypes.MapType(keyType, valueType);
- types.put(dT.getName(), dT);
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.MAP, dT.getName());
- return dT;
- }
-
- public EnumType defineEnumType(String name, EnumValue... values) throws MetadataException {
- return defineEnumType(new EnumTypeDefinition(name, values));
- }
-
- public EnumType defineEnumType(EnumTypeDefinition eDef) throws MetadataException {
- assert eDef.name != null;
- if (types.containsKey(eDef.name)) {
- throw new MetadataException(
- String.format("Redefinition of type %s not supported", eDef.name));
- }
-
- EnumType eT = new EnumType(this, eDef.name, eDef.enumValues);
- types.put(eDef.name, eT);
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ENUM, eDef.name);
- return eT;
- }
-
- public SimpleDateFormat getDateFormat() {
- return dateFormat.get();
- }
-
- public boolean allowNullsInCollections() {
- return false;
- }
-
- public void removeTypes(ImmutableList<String> typeNames) {
-
- }
-
- class TransientTypeSystem extends TypeSystem {
-
- final ImmutableList<StructTypeDefinition> structDefs;
- final ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs;
- final ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs;
- Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<>();
- Map<String, HierarchicalTypeDefinition<TraitType>> traitNameToDefMap =
- new HashMap<>();
- Map<String, HierarchicalTypeDefinition<ClassType>> classNameToDefMap =
- new HashMap<>();
-
- Set<String> transientTypes;
-
- List<AttributeInfo> recursiveRefs;
- List<DataTypes.ArrayType> recursiveArrayTypes;
- List<DataTypes.MapType> recursiveMapTypes;
-
-
- TransientTypeSystem(ImmutableList<StructTypeDefinition> structDefs,
- ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
- ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) {
-
- this.structDefs = structDefs;
- this.traitDefs = traitDefs;
- this.classDefs = classDefs;
- structNameToDefMap = new HashMap<>();
- traitNameToDefMap = new HashMap<>();
- classNameToDefMap = new HashMap<>();
-
- recursiveRefs = new ArrayList<>();
- recursiveArrayTypes = new ArrayList<>();
- recursiveMapTypes = new ArrayList<>();
- transientTypes = new LinkedHashSet<>();
- }
-
- private IDataType dataType(String name) {
- return TypeSystem.this.types.get(name);
- }
-
- /*
- * Step 1:
- * - validate cannot redefine types
- * - setup shallow Type instances to facilitate recursive type graphs
- */
- private void step1() throws MetadataException {
- for (StructTypeDefinition sDef : structDefs) {
- assert sDef.typeName != null;
- if (dataType(sDef.typeName) != null) {
- throw new MetadataException(
- String.format("Cannot redefine type %s", sDef.typeName));
- }
- TypeSystem.this.types.put(sDef.typeName,
- new StructType(this, sDef.typeName, sDef.attributeDefinitions.length));
- structNameToDefMap.put(sDef.typeName, sDef);
- transientTypes.add(sDef.typeName);
- }
-
- for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) {
- assert traitDef.typeName != null;
- if (types.containsKey(traitDef.typeName)) {
- throw new MetadataException(
- String.format("Cannot redefine type %s", traitDef.typeName));
- }
-
- TypeSystem.this.types.put(traitDef.typeName,
- new TraitType(this, traitDef.typeName, traitDef.superTypes,
- traitDef.attributeDefinitions.length));
- traitNameToDefMap.put(traitDef.typeName, traitDef);
- transientTypes.add(traitDef.typeName);
- }
-
- for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) {
- assert classDef.typeName != null;
- if (types.containsKey(classDef.typeName)) {
- throw new MetadataException(
- String.format("Cannot redefine type %s", classDef.typeName));
- }
-
- TypeSystem.this.types.put(classDef.typeName,
- new ClassType(this, classDef.typeName, classDef.superTypes,
- classDef.attributeDefinitions.length));
- classNameToDefMap.put(classDef.typeName, classDef);
- transientTypes.add(classDef.typeName);
- }
- }
-
- private <U extends HierarchicalType> void validateSuperTypes(Class<U> cls,
- HierarchicalTypeDefinition<U> def)
- throws MetadataException {
- Set<String> s = new HashSet<>();
- ImmutableList<String> superTypes = def.superTypes;
- for (String superTypeName : superTypes) {
-
- if (s.contains(superTypeName)) {
- throw new MetadataException(
- String.format("Type %s extends superType %s multiple times",
- def.typeName, superTypeName));
- }
-
- IDataType dT = dataType(superTypeName);
-
- if (dT == null) {
- throw new MetadataException(
- String.format("Unknown superType %s in definition of type %s",
- superTypeName, def.typeName));
- }
-
- if (!cls.isAssignableFrom(dT.getClass())) {
- throw new MetadataException(
- String.format("SuperType %s must be a %s, in definition of type %s",
- superTypeName, cls.getName(), def.typeName));
- }
- s.add(superTypeName);
- }
- }
-
- /*
- * Step 2:
- * - for Hierarchical Types, validate SuperTypes.
- * - for each Hierarchical Type setup their SuperTypes Graph
- */
- private void step2() throws MetadataException {
- for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) {
- validateSuperTypes(TraitType.class, traitDef);
- TraitType traitType = getDataType(TraitType.class, traitDef.typeName);
- traitType.setupSuperTypesGraph();
- }
-
- for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) {
- validateSuperTypes(ClassType.class, classDef);
- ClassType classType = getDataType(ClassType.class, classDef.typeName);
- classType.setupSuperTypesGraph();
- }
- }
-
- private AttributeInfo constructAttributeInfo(AttributeDefinition attrDef)
- throws MetadataException {
- AttributeInfo info = new AttributeInfo(this, attrDef, null);
- if (transientTypes.contains(attrDef.dataTypeName)) {
- recursiveRefs.add(info);
- }
- if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- DataTypes.ArrayType arrType = (DataTypes.ArrayType) info.dataType();
- if (transientTypes.contains(arrType.getElemType().getName())) {
- recursiveArrayTypes.add(arrType);
- }
- }
- if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
- DataTypes.MapType mapType = (DataTypes.MapType) info.dataType();
- if (transientTypes.contains(mapType.getKeyType().getName())) {
- recursiveMapTypes.add(mapType);
- } else if (transientTypes.contains(mapType.getValueType().getName())) {
- recursiveMapTypes.add(mapType);
- }
- }
-
- if (info.multiplicity.upper > 1 && !(
- info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP ||
- info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY)) {
- throw new MetadataException(
- String.format("A multiplicty of more than one requires a collection type for attribute '%s'",
- info.name));
- }
-
- return info;
- }
-
- private StructType constructStructureType(StructTypeDefinition def)
- throws MetadataException {
- AttributeInfo[] infos = new AttributeInfo[def.attributeDefinitions.length];
- for (int i = 0; i < def.attributeDefinitions.length; i++) {
- infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
- }
-
- StructType type = new StructType(TypeSystem.this, def.typeName, null, infos);
- TypeSystem.this.types.put(def.typeName, type);
- return type;
- }
-
- private <U extends HierarchicalType> U constructHierarchicalType(Class<U> cls,
- HierarchicalTypeDefinition<U> def)
- throws MetadataException {
- AttributeInfo[] infos = new AttributeInfo[def.attributeDefinitions.length];
- for (int i = 0; i < def.attributeDefinitions.length; i++) {
- infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
- }
-
- try {
- Constructor<U> cons = cls.getDeclaredConstructor(
- TypeSystem.class,
- String.class,
- ImmutableList.class,
- AttributeInfo[].class);
- U type = cons.newInstance(TypeSystem.this, def.typeName, def.superTypes, infos);
- TypeSystem.this.types.put(def.typeName, type);
- return type;
- } catch (Exception e) {
- throw new MetadataException(
- String.format("Cannot construct Type of MetaType %s", cls.getName()), e);
- }
- }
-
- /*
- * Step 3:
- * - Order Hierarchical Types in order of SuperType before SubType.
- * - Construct all the Types
- */
- private void step3() throws MetadataException {
-
- List<TraitType> traitTypes = new ArrayList<>();
- for (String traitTypeName : traitNameToDefMap.keySet()) {
- traitTypes.add(getDataType(TraitType.class, traitTypeName));
- }
- Collections.sort(traitTypes);
-
- List<ClassType> classTypes = new ArrayList<>();
- for (String classTypeName : classNameToDefMap.keySet()) {
- classTypes.add(getDataType(ClassType.class, classTypeName));
- }
- Collections.sort(classTypes);
-
- for (StructTypeDefinition structDef : structDefs) {
- constructStructureType(structDef);
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.CLASS, structDef.typeName);
- }
-
- for (TraitType traitType : traitTypes) {
- constructHierarchicalType(TraitType.class,
- traitNameToDefMap.get(traitType.getName()));
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.TRAIT, traitType.getName());
- }
-
- for (ClassType classType : classTypes) {
- constructHierarchicalType(ClassType.class,
- classNameToDefMap.get(classType.getName()));
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.CLASS, classType.getName());
- }
- }
-
- /*
- * Step 4:
- * - fix up references in recursive AttrInfo and recursive Collection Types.
- */
- private void step4() throws MetadataException {
- for (AttributeInfo info : recursiveRefs) {
- info.setDataType(dataType(info.dataType().getName()));
- }
- for (DataTypes.ArrayType arrType : recursiveArrayTypes) {
- arrType.setElemType(dataType(arrType.getElemType().getName()));
- }
- for (DataTypes.MapType mapType : recursiveMapTypes) {
- mapType.setKeyType(dataType(mapType.getKeyType().getName()));
- mapType.setValueType(dataType(mapType.getValueType().getName()));
- }
- }
-
- Map<String, IDataType> defineTypes() throws MetadataException {
- step1();
- step2();
- try {
- step3();
- step4();
- } catch (MetadataException me) {
- for (String sT : transientTypes) {
- types.remove(sT);
- }
- throw me;
- }
-
- Map<String, IDataType> newTypes = new HashMap<>();
-
- for (String tName : transientTypes) {
- newTypes.put(tName, dataType(tName));
- }
- return newTypes;
- }
-
- @Override
- public ImmutableList<String> getTypeNames() {
- return TypeSystem.this.getTypeNames();
- }
-
- @Override
- public <T> T getDataType(Class<T> cls, String name) throws MetadataException {
- return TypeSystem.this.getDataType(cls, name);
- }
-
- @Override
- public StructType defineStructType(String name, boolean errorIfExists,
- AttributeDefinition... attrDefs)
- throws MetadataException {
- throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
- }
-
- @Override
- public TraitType defineTraitType(HierarchicalTypeDefinition traitDef)
- throws MetadataException {
- throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
- }
-
- @Override
- public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef
- ) throws MetadataException {
- throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
- }
-
- @Override
- public Map<String, IDataType> defineTypes(ImmutableList<StructTypeDefinition> structDefs,
- ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
- ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs)
- throws MetadataException {
- throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
- }
-
- @Override
- public DataTypes.ArrayType defineArrayType(IDataType elemType) throws MetadataException {
- throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
- }
-
- @Override
- public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType)
- throws MetadataException {
- throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
- }
- }
-
- public class IdType {
- private static final String ID_ATTRNAME = "guid";
- private static final String TYPENAME_ATTRNAME = "typeName";
- private static final String TYP_NAME = "__IdType";
-
- private IdType() {
- AttributeDefinition idAttr = new AttributeDefinition(ID_ATTRNAME,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null);
- AttributeDefinition typNmAttr =
- new AttributeDefinition(TYPENAME_ATTRNAME,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null);
- try {
- AttributeInfo[] infos = new AttributeInfo[2];
- infos[0] = new AttributeInfo(TypeSystem.this, idAttr, null);
- infos[1] = new AttributeInfo(TypeSystem.this, typNmAttr, null);
-
- StructType type = new StructType(TypeSystem.this, TYP_NAME, null, infos);
- TypeSystem.this.types.put(TYP_NAME, type);
-
- } catch (MetadataException me) {
- throw new RuntimeException(me);
- }
- }
-
- public StructType getStructType() throws MetadataException {
- return getDataType(StructType.class, TYP_NAME);
- }
-
- public String getName() { return TYP_NAME; }
- public String idAttrName() { return ID_ATTRNAME;}
- public String typeNameAttrName() { return TYPENAME_ATTRNAME;}
- }
-
- public static final String ID_STRUCT_ID_ATTRNAME = IdType.ID_ATTRNAME;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeUtils.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeUtils.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeUtils.java
deleted file mode 100755
index f494c4a..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeUtils.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import scala.collection.JavaConversions;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class TypeUtils {
-
- public static final String NAME_REGEX = "[a-zA-z][a-zA-Z0-9_]*";
- public static final Pattern NAME_PATTERN = Pattern.compile(NAME_REGEX);
- public static final Pattern ARRAY_TYPE_NAME_PATTERN = Pattern
- .compile(String.format("array<(%s)>", NAME_REGEX));
- public static final Pattern MAP_TYPE_NAME_PATTERN =
- Pattern.compile(String.format("map<(%s),(%s)>", NAME_REGEX, NAME_REGEX));
-
- public static void outputVal(String val, Appendable buf, String prefix)
- throws MetadataException {
- try {
- buf.append(prefix).append(val);
- } catch (IOException ie) {
- throw new MetadataException(ie);
- }
- }
-
- public static String parseAsArrayType(String typeName) {
- Matcher m = ARRAY_TYPE_NAME_PATTERN.matcher(typeName);
- return m.matches() ? m.group(1) : null;
- }
-
- public static String[] parseAsMapType(String typeName) {
- Matcher m = MAP_TYPE_NAME_PATTERN.matcher(typeName);
- return m.matches() ? new String[]{m.group(1), m.group(2)} : null;
- }
-
- public static Map<AttributeInfo, List<String>> buildAttrInfoToNameMap(FieldMapping f) {
- Map<AttributeInfo, List<String>> b = new HashMap<AttributeInfo, List<String>>();
- for (Map.Entry<String, AttributeInfo> e : f.fields.entrySet()) {
- List<String> names = b.get(e.getValue());
- if (names == null) {
- names = new ArrayList<String>();
- b.put(e.getValue(), names);
- }
- names.add(e.getKey());
- }
- return ImmutableMap.copyOf(b);
- }
-
- public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums, ImmutableList<StructTypeDefinition> structs,
- ImmutableList<HierarchicalTypeDefinition<TraitType>> traits,
- ImmutableList<HierarchicalTypeDefinition<ClassType>> classes) {
- return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
- JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
- }
-
- protected static class Pair<L,R> {
- protected L left;
- protected R right;
-
- public Pair(L left, R right) {
- this.left = left;
- this.right = right;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypedStructHandler.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypedStructHandler.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypedStructHandler.java
deleted file mode 100755
index c0e1c56..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypedStructHandler.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-
-public class TypedStructHandler {
-
- private final IConstructableType<IStruct, ITypedStruct> structType;
- private final FieldMapping fieldMapping;
-
- public TypedStructHandler(IConstructableType<IStruct, ITypedStruct> structType) {
- this.structType = structType;
- fieldMapping = structType.fieldMapping();
- }
-
- public ITypedStruct convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof ITypedStruct) {
- ITypedStruct ts = (ITypedStruct) val;
- if (ts.getTypeName() != structType.getName()) {
- throw new ValueConversionException(structType, val);
- }
- return ts;
- } else if (val instanceof Struct) {
- Struct s = (Struct) val;
- if (!s.typeName.equals(structType.getName())) {
- throw new ValueConversionException(structType, val);
- }
- ITypedStruct ts = createInstance();
- for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
- String attrKey = e.getKey();
- AttributeInfo i = e.getValue();
- Object aVal = s.get(attrKey);
- try {
- ts.set(attrKey, aVal);
- } catch (ValueConversionException ve) {
- throw new ValueConversionException(structType, val, ve);
- }
- }
- return ts;
- } else if (val instanceof StructInstance &&
- ((StructInstance) val).getTypeName() == structType.getName()) {
- return (StructInstance) val;
- } else {
- throw new ValueConversionException(structType, val);
- }
- }
- if (!m.nullAllowed()) {
- throw new ValueConversionException.NullConversionException(m);
- }
- return null;
- }
-
- public DataTypes.TypeCategory getTypeCategory() {
- return DataTypes.TypeCategory.STRUCT;
- }
-
- public ITypedStruct createInstance() {
- return new StructInstance(structType.getName(),
- fieldMapping,
- new boolean[fieldMapping.fields.size()],
- fieldMapping.numBools == 0 ? null : new boolean[fieldMapping.numBools],
- fieldMapping.numBytes == 0 ? null : new byte[fieldMapping.numBytes],
- fieldMapping.numShorts == 0 ? null : new short[fieldMapping.numShorts],
- fieldMapping.numInts == 0 ? null : new int[fieldMapping.numInts],
- fieldMapping.numLongs == 0 ? null : new long[fieldMapping.numLongs],
- fieldMapping.numFloats == 0 ? null : new float[fieldMapping.numFloats],
- fieldMapping.numDoubles == 0 ? null : new double[fieldMapping.numDoubles],
- fieldMapping.numBigDecimals == 0 ? null
- : new BigDecimal[fieldMapping.numBigDecimals],
- fieldMapping.numBigInts == 0 ? null : new BigInteger[fieldMapping.numBigInts],
- fieldMapping.numDates == 0 ? null : new Date[fieldMapping.numDates],
- fieldMapping.numStrings == 0 ? null : new String[fieldMapping.numStrings],
- fieldMapping.numArrays == 0 ? null : new ImmutableList[fieldMapping.numArrays],
- fieldMapping.numMaps == 0 ? null : new ImmutableMap[fieldMapping.numMaps],
- fieldMapping.numStructs == 0 ? null : new StructInstance[fieldMapping.numStructs],
- fieldMapping.numReferenceables == 0 ? null
- : new ReferenceableInstance[fieldMapping.numReferenceables],
- fieldMapping.numReferenceables == 0 ? null
- : new Id[fieldMapping.numReferenceables]);
- }
-
- public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
- fieldMapping.output(s, buf, prefix);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ValueConversionException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ValueConversionException.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ValueConversionException.java
deleted file mode 100755
index 031f8be..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ValueConversionException.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-public class ValueConversionException extends MetadataException {
-
- public ValueConversionException(IDataType typ, Object val) {
- this(typ, val, (Throwable) null);
- }
-
- public ValueConversionException(IDataType typ, Object val, Throwable t) {
- super(String
- .format("Cannot convert value '%s' to datatype %s", val.toString(), typ.getName()),
- t);
- }
-
- public ValueConversionException(IDataType typ, Object val, String msg) {
- super(String.format("Cannot convert value '%s' to datatype %s because: %s",
- val.toString(), typ.getName(), msg));
- }
-
- public ValueConversionException(String typeName, Object val, String msg) {
- super(String.format("Cannot convert value '%s' to datatype %s because: %s",
- val.toString(), typeName, msg));
- }
-
- protected ValueConversionException(String msg) {
- super(msg);
- }
-
- public static class NullConversionException extends ValueConversionException {
- public NullConversionException(Multiplicity m) {
- super(String.format("Null value not allowed for multiplicty %s", m));
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/package-info.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/package-info.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/package-info.java
deleted file mode 100755
index 4cf9967..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/package-info.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <h2>Types:</h2>
- * <img src="doc-files/dataTypes.png" />
- * <ul>
- * <li> <b>IDataType:</b> Represents a <i>DataType</i> in the TypeSystem. All Instances and
- * Attributes are associated
- * with a DataType. They represent the <b>Set</b> of values that Instances/Attributes of this
- * type can have.
- * Currently the namespace of DataTypes is flat. DataTypes can be asked to <i>convert</i>
- * arbitrary java Objects
- * to instances of this type, and they can be asked for a String representation of an
- * instance.</li>
- * <li><b>Type Categories:</b></li> DataTypes are grouped into Categories. A Category implies
- * certain semantics about
- * the Types belonging to the Category. We have PRIMITIVE, ENUM, ARRAY, MAP, STRUCT, TRAIT,
- * and CLASS categories.
- * <li><b>Primitive Types:</b> There are corresponding DataTypes for the java primitives:
- * Boolean, Byte, Short,
- * Int, Long, Float, Double. We also support BigInteger, BigDecimal, String, and Date</li>
- * <li><b>Collection Types:</b>ArrayType and MapType are parameterized DataTypes taking one
- * and two parameters
- * respectively.</li>
- * <li><b>Enum Types:</b> Used to define DataTypes with all valid values listed in the Type
- * definition. For e.g.
- * <pre>
- * {@code
- * ts.defineEnumType("HiveObjectType",
-new EnumValue("GLOBAL", 1),
-new EnumValue("DATABASE", 2),
-new EnumValue("TABLE", 3),
-new EnumValue("PARTITION", 4),
-new EnumValue("COLUMN", 5))
- * }
- * </pre> Each <i>EnumValue</i> has name and an ordinal. Either one can be used as a value for an
- * Attribute of this Type.
- * </li>
- * <li><b>Constructable Types:</b> Are complex Types that are composed of Attributes. We
- * support Structs, Classes
- * and Traits constructable types. A ConstructableType is parameterized by the Type of its
- * <i>Instance</i> java
- * class(these are implementations of the ITypedInstance interface). A value of the
- * IConstructableType will
- * implement this parameterized Type. IConstructableTypes can be asked to create an 'empty'
- * instance of their Type.
- * IConstructableTypes are associated with FieldMappings that encapsulate the mapping from/to
- * the ITypedInstance
- * java object.
- * </li>
- * <li><b>Attribute Info:</b>Represents an Attribute of a complex datatype. Attributes are
- * defined by a name, a
- * dataType, its Multiplicity and whether it is a composite relation. <i>Multiplicity</i> is
- * a constraint on the
- * number of instances that an instance can have. For non collection types and Maps:
- * Multiplicity is OPTIONAL or
- * REQUIRED.
- * For Arrays the Multiplicity is specified by a lower-bound, upper-bound and a uniqueness
- * constraint.
- * </li>
- * <li><b>Struct Types:</b>Are IConstructableTypes whose instances are IStructs. Conceptually
- * these are like 'C'
- * structs: they represent a collection of Attributes. For e.g.
- * <pre>
- * {@code
- * ts.defineStructType(STRUCT_TYPE_1,
-true,
-createRequiredAttrDef("a", DataTypes.INT_TYPE),
-createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
-createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
-createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
-createOptionalAttrDef("e", DataTypes.INT_TYPE),
-createOptionalAttrDef("f", DataTypes.INT_TYPE),
-createOptionalAttrDef("g", DataTypes.LONG_TYPE),
-createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
-createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
-createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
-createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
-createOptionalAttrDef("l", DataTypes.DATE_TYPE),
-createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
-createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
-createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE))
- * }
- * </pre>
- * </li>
- * <li><b>Hierarchical Types:</b>Are DataTypes that can have a SuperType. Classes and Traits
- * are the supported
- * Hierarchical Types. </li>
- * <li><b>Class Types:</b></li>
- * <li><b>Trait Types:</b></li>
- * </ul>
- *
- *
- * <h2>Instances:</h2>
- * <img src="doc-files/instance.png" />
- * <ul>
- * <li> <b>IStruct:</b></li>
- * <li><b>IReferenceableInstance:</b></li>
- * <li><b>ITypedStruct:</b></li>
- * <li><b>ITypedReferenceableInstance:</b></li>
- * </ul>
- *
- * <h3>Serialization of Types:</h3>
- *
- * <h3>Serialization of Instances:</h3>
- *
- * <h3>Searching on Classes and Traits:</h3>
- */
-package org.apache.hadoop.metadata.typesystem.types;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/utils/TypesUtil.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/utils/TypesUtil.java
deleted file mode 100755
index 4c2d3d6..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/utils/TypesUtil.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types.utils;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-
-/**
- * Types utilities class.
- */
-public class TypesUtil {
-
- private TypesUtil() {
- }
-
- public static AttributeDefinition createOptionalAttrDef(String name,
- IDataType dataType) {
- return new AttributeDefinition(name, dataType.getName(),
- Multiplicity.OPTIONAL, false, null);
- }
-
- public static AttributeDefinition createOptionalAttrDef(String name,
- String dataType) {
- return new AttributeDefinition(name, dataType, Multiplicity.OPTIONAL, false, null);
- }
-
- public static AttributeDefinition createRequiredAttrDef(String name,
- String dataType) {
- return new AttributeDefinition(name, dataType, Multiplicity.REQUIRED, false, null);
- }
-
- public static AttributeDefinition createUniqueRequiredAttrDef(String name,
- IDataType dataType) {
- return new AttributeDefinition(name, dataType.getName(),
- Multiplicity.REQUIRED, false, true, true, null);
- }
-
- public static AttributeDefinition createRequiredAttrDef(String name,
- IDataType dataType) {
- return new AttributeDefinition(name, dataType.getName(),
- Multiplicity.REQUIRED, false, null);
- }
-
- public static EnumTypeDefinition createEnumTypeDef(String name, EnumValue... enumValues) {
- return new EnumTypeDefinition(name, enumValues);
- }
-
- public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(
- String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
- return new HierarchicalTypeDefinition<>(TraitType.class, name, superTypes, attrDefs);
- }
-
- public static StructTypeDefinition createStructTypeDef(String name, AttributeDefinition... attrDefs) {
- return new StructTypeDefinition(name, attrDefs);
- }
-
- public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(
- String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
- return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/atlas/typesystem/TypesDef.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/TypesDef.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/TypesDef.scala
new file mode 100755
index 0000000..b51048d
--- /dev/null
+++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/TypesDef.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem
+
+import org.apache.atlas.typesystem.types._
+
+case class TypesDef(enumTypes: Seq[EnumTypeDefinition],
+ structTypes: Seq[StructTypeDefinition],
+ traitTypes: Seq[HierarchicalTypeDefinition[TraitType]],
+ classTypes: Seq[HierarchicalTypeDefinition[ClassType]]) {
+ def this() = this(Seq(), Seq(), Seq(), Seq())
+ def this(enumType : EnumTypeDefinition) = this(Seq(enumType), Seq(), Seq(), Seq())
+ def this(structType: StructTypeDefinition) = this(Seq(), Seq(structType), Seq(), Seq())
+ def this(typ: HierarchicalTypeDefinition[_], isTrait : Boolean) = this(
+ Seq(),
+ Seq(),
+ if ( isTrait )
+ Seq(typ.asInstanceOf[HierarchicalTypeDefinition[TraitType]]) else Seq(),
+ if (!isTrait )
+ Seq(typ.asInstanceOf[HierarchicalTypeDefinition[ClassType]]) else Seq()
+ )
+
+ def enumTypesAsJavaList() = {
+ import scala.collection.JavaConverters._
+ enumTypes.asJava
+ }
+
+ def structTypesAsJavaList() = {
+ import scala.collection.JavaConverters._
+ structTypes.asJava
+ }
+
+ def traitTypesAsJavaList() = {
+ import scala.collection.JavaConverters._
+ traitTypes.asJava
+ }
+
+ def classTypesAsJavaList() = {
+ import scala.collection.JavaConverters._
+ classTypes.asJava
+ }
+
+ def isEmpty() = {
+ enumTypes.isEmpty & structTypes.isEmpty & traitTypes.isEmpty & classTypes.isEmpty
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/InstanceBuilder.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/InstanceBuilder.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/InstanceBuilder.scala
new file mode 100644
index 0000000..df1851c
--- /dev/null
+++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/InstanceBuilder.scala
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.builders
+
+import org.apache.atlas.typesystem.{IReferenceableInstance, IStruct, Referenceable, Struct}
+
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.language.{dynamics, implicitConversions}
+import scala.util.DynamicVariable
+
+class InstanceBuilder extends Dynamic {
+
+ private val references : ArrayBuffer[Referenceable] = new ArrayBuffer[Referenceable]()
+
+ val context = new DynamicVariable[DynamicStruct](null)
+
+ def struct(typeName : String) : DynamicStruct = {
+ context.value = new DynamicStruct(this, new Struct(typeName))
+ context.value
+ }
+
+ def instance(typeName: String, traitNames: String*)(f : => Unit) : DynamicReference = {
+ val r = new Referenceable(typeName, traitNames:_*)
+ references.append(r)
+ val dr = new DynamicReference(this, r)
+ context.withValue(dr){f}
+ dr
+ }
+
+ def create( f : => Unit ) : java.util.List[Referenceable] = {
+ f
+ references.asJava
+ }
+
+ def applyDynamic(name : String)(value : Any) : Any = {
+ context.value.updateDynamic(name)(value)
+ }
+
+ implicit def symbolToDynamicStruct(s : Symbol) : DynamicValue =
+ new DynamicValue(this, s.name, if (context.value == null) null else context.value.s)
+
+}
+
+object DynamicValue {
+
+ private[builders] def transformOut(s: IStruct, attr : String, v : Any)(implicit ib : InstanceBuilder) : DynamicValue =
+ v match {
+ case r : Referenceable => new DynamicReference(ib, r)
+ case s : Struct => new DynamicStruct(ib, s)
+ case jL : java.util.List[_] => {
+ if ( s != null ) {
+ new DynamicCollection(ib, attr, s)
+ } else {
+ new DynamicValue(ib, attr, s, jL.map{ e => transformOut(null, null, e)}.toSeq)
+ }
+ }
+ case jM : java.util.Map[_,_] => {
+ if ( s != null ) {
+ new DynamicMap(ib, attr, s)
+ } else {
+ new DynamicValue(ib, attr, s, jM.map {
+ case (k, v) => k -> transformOut(null, null, v)
+ }.toMap)
+ }
+ }
+ case x => {
+ if ( s != null ) {
+ new DynamicValue(ib, attr, s)
+ } else {
+ new DynamicValue(ib, attr, s, x)
+ }
+ }
+ }
+
+ private[builders] def transformIn(v : Any) : Any = v match {
+ case dr : DynamicReference => dr.r
+ case ds : DynamicStruct => ds.s
+ case dv : DynamicValue => dv.get
+ case l : Seq[_] => l.map{ e => transformIn(e)}.asJava
+ case m : Map[_,_] => m.map {
+ case (k,v) => k -> transformIn(v)
+ }.asJava
+ case x => x
+ }
+
+}
+
+class DynamicValue(val ib : InstanceBuilder, val attrName : String, val s: IStruct, var value : Any = null) extends Dynamic {
+ import DynamicValue._
+
+ implicit val iib : InstanceBuilder = ib
+
+ def ~(v : Any): Unit = {
+ if ( s != null ) {
+ s.set(attrName, transformIn(v))
+ } else {
+ value = v
+ }
+ }
+
+ def get : Any = if ( s != null ) s.get(attrName) else value
+
+ def selectDynamic(name: String) : DynamicValue = {
+
+ throw new UnsupportedOperationException()
+ }
+
+ def update(key : Any, value : Object): Unit = {
+ throw new UnsupportedOperationException()
+ }
+
+ def apply(key : Any): DynamicValue = {
+
+ if ( s != null && s.isInstanceOf[Referenceable] && key.isInstanceOf[String]) {
+ val r = s.asInstanceOf[Referenceable]
+ if ( r.getTraits contains attrName ) {
+ val traitAttr = key.asInstanceOf[String]
+ return new DynamicStruct(ib, r.getTrait(attrName)).selectDynamic(traitAttr)
+ }
+ }
+ throw new UnsupportedOperationException()
+ }
+}
+
+class DynamicCollection(ib : InstanceBuilder, attrName : String, s: IStruct) extends DynamicValue(ib, attrName ,s) {
+ import DynamicValue._
+
+ override def update(key : Any, value : Object): Unit = {
+ var jL = s.get(attrName)
+ val idx = key.asInstanceOf[Int]
+ if (jL == null ) {
+ val l = new java.util.ArrayList[Object]()
+ l.ensureCapacity(idx)
+ jL = l
+ }
+ val nJL = new java.util.ArrayList[Object](jL.asInstanceOf[java.util.List[Object]])
+ nJL.asInstanceOf[java.util.List[Object]].set(idx, transformIn(value).asInstanceOf[Object])
+ s.set(attrName, nJL)
+ }
+
+ override def apply(key : Any): DynamicValue = {
+ var jL = s.get(attrName)
+ val idx = key.asInstanceOf[Int]
+ if (jL == null ) {
+ null
+ } else {
+ transformOut(null, null, jL.asInstanceOf[java.util.List[Object]].get(idx))
+ }
+ }
+}
+
+class DynamicMap(ib : InstanceBuilder, attrName : String, s: IStruct) extends DynamicValue(ib, attrName ,s) {
+ import DynamicValue._
+ override def update(key : Any, value : Object): Unit = {
+ var jM = s.get(attrName)
+ if (jM == null ) {
+ jM = new java.util.HashMap[Object, Object]()
+ }
+ jM.asInstanceOf[java.util.Map[Object, Object]].put(key.asInstanceOf[AnyRef], value)
+ }
+
+ override def apply(key : Any): DynamicValue = {
+ var jM = s.get(attrName)
+ if (jM == null ) {
+ null
+ } else {
+ transformOut(null, null, jM.asInstanceOf[java.util.Map[Object, Object]].get(key))
+ }
+ }
+}
+
+class DynamicStruct(ib : InstanceBuilder, s: IStruct) extends DynamicValue(ib, null ,s) {
+ import DynamicValue._
+ override def selectDynamic(name: String) : DynamicValue = {
+ transformOut(s, name, s.get(name))
+ }
+
+ def updateDynamic(name: String)(value: Any) {
+ s.set(name, transformIn(value))
+ }
+
+ override def ~(v : Any): Unit = { throw new UnsupportedOperationException()}
+ override def get : Any = s
+
+}
+
+class DynamicReference(ib : InstanceBuilder, val r : IReferenceableInstance) extends DynamicStruct(ib, r) {
+
+ private def _trait(name : String) = new DynamicStruct(ib, r.getTrait(name))
+
+ override def selectDynamic(name: String) : DynamicValue = {
+ if ( r.getTraits contains name ) {
+ _trait(name)
+ } else {
+ super.selectDynamic(name)
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala
new file mode 100644
index 0000000..cd711d6
--- /dev/null
+++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/builders/TypesBuilder.scala
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.builders
+
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.typesystem.TypesDef
+import org.apache.atlas.typesystem.types._
+import org.apache.atlas.typesystem.types.utils.TypesUtil
+
+import scala.collection.mutable.ArrayBuffer
+import scala.language.{dynamics, implicitConversions, postfixOps}
+import scala.util.DynamicVariable
+
+object TypesBuilder {
+
+ case class Context(enums : ArrayBuffer[EnumTypeDefinition],
+ structs : ArrayBuffer[StructTypeDefinition],
+ classes : ArrayBuffer[HierarchicalTypeDefinition[ClassType]],
+ traits : ArrayBuffer[HierarchicalTypeDefinition[TraitType]],
+ currentTypeAttrs : ArrayBuffer[Attr] = null)
+
+ class AttrOption()
+ class ReverseAttributeName(val rNm : String) extends AttrOption
+ class MultiplicityOption(val lower: Int, val upper: Int, val isUnique: Boolean) extends AttrOption
+
+ val required = new AttrOption()
+ val optional = new AttrOption()
+ val collection = new AttrOption()
+ val set = new AttrOption()
+ val composite = new AttrOption()
+ val unique = new AttrOption()
+ val indexed = new AttrOption()
+ def reverseAttributeName(rNm : String) = new ReverseAttributeName(rNm)
+ def multiplicty(lower: Int, upper: Int, isUnique: Boolean) = new MultiplicityOption(lower, upper, isUnique)
+
+ val boolean = DataTypes.BOOLEAN_TYPE.getName
+ val byte = DataTypes.BYTE_TYPE.getName
+ val short = DataTypes.SHORT_TYPE.getName
+ val int = DataTypes.INT_TYPE.getName
+ val long = DataTypes.LONG_TYPE.getName
+ val float = DataTypes.FLOAT_TYPE.getName
+
+ val double = DataTypes.DOUBLE_TYPE.getName
+ val bigint = DataTypes.BIGINTEGER_TYPE.getName
+ val bigdecimal = DataTypes.BIGDECIMAL_TYPE.getName
+ val date = DataTypes.DATE_TYPE.getName
+ val string = DataTypes.STRING_TYPE.getName
+
+ def array(t : String) : String = {
+ DataTypes.arrayTypeName(t)
+ }
+
+ def map(kt : String, vt : String) : String = {
+ DataTypes.mapTypeName(kt, vt)
+ }
+
+ class Attr(ctx : Context, val name : String) {
+
+ private var dataTypeName : String = DataTypes.BOOLEAN_TYPE.getName
+ private var multiplicity: Multiplicity = Multiplicity.OPTIONAL
+ private var isComposite: Boolean = false
+ private var reverseAttributeName: String = null
+ private var isUnique: Boolean = false
+ private var isIndexable: Boolean = false
+
+ ctx.currentTypeAttrs += this
+
+ def getDef : AttributeDefinition =
+ new AttributeDefinition(name, dataTypeName,
+ multiplicity, isComposite, isUnique, isIndexable, reverseAttributeName)
+
+ def `~`(dT : String, options : AttrOption*) : Attr = {
+ dataTypeName = dT
+ options.foreach { o =>
+ o match {
+ case `required` => {multiplicity = Multiplicity.REQUIRED}
+ case `optional` => {multiplicity = Multiplicity.OPTIONAL}
+ case `collection` => {multiplicity = Multiplicity.COLLECTION}
+ case `set` => {multiplicity = Multiplicity.SET}
+ case `composite` => {isComposite = true}
+ case `unique` => {isUnique = true}
+ case `indexed` => {isIndexable = true}
+ case m : MultiplicityOption => {multiplicity = new Multiplicity(m.lower, m.upper, m.isUnique)}
+ case r : ReverseAttributeName => {reverseAttributeName = r.rNm}
+ case _ => ()
+ }
+ }
+ this
+ }
+
+ }
+
+}
+
+class TypesBuilder {
+
+ import org.apache.atlas.typesystem.builders.TypesBuilder.{Attr, Context}
+
+ val required = TypesBuilder.required
+ val optional = TypesBuilder.optional
+ val collection = TypesBuilder.collection
+ val set = TypesBuilder.set
+ val composite = TypesBuilder.composite
+ val unique = TypesBuilder.unique
+ val indexed = TypesBuilder.indexed
+ def multiplicty = TypesBuilder.multiplicty _
+ def reverseAttributeName = TypesBuilder.reverseAttributeName _
+
+ val boolean = TypesBuilder.boolean
+ val byte = TypesBuilder.byte
+ val short = TypesBuilder.short
+ val int = TypesBuilder.int
+ val long = TypesBuilder.long
+ val float = TypesBuilder.float
+
+ val double = TypesBuilder.double
+ val bigint = TypesBuilder.bigint
+ val bigdecimal = TypesBuilder.bigdecimal
+ val date = TypesBuilder.date
+ val string = TypesBuilder.string
+
+ def array = TypesBuilder.array _
+
+ def map = TypesBuilder.map _
+
+ val context = new DynamicVariable[Context](Context(new ArrayBuffer(),
+ new ArrayBuffer(),
+ new ArrayBuffer(),
+ new ArrayBuffer()))
+
+ implicit def strToAttr(s : String) = new Attr(context.value, s)
+
+ def types(f : => Unit ) : TypesDef = {
+ f
+ TypesDef(context.value.enums.toSeq,
+ context.value.structs.toSeq,
+ context.value.traits.toSeq,
+ context.value.classes.toSeq)
+ }
+
+ def _class(name : String, superTypes : List[String] = List())(f : => Unit): Unit = {
+ val attrs = new ArrayBuffer[Attr]()
+ context.withValue(context.value.copy(currentTypeAttrs = attrs)){f}
+ context.value.classes +=
+ TypesUtil.createClassTypeDef(name, ImmutableList.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*)
+ }
+
+ def _trait(name : String, superTypes : List[String] = List())(f : => Unit): Unit = {
+ val attrs = new ArrayBuffer[Attr]()
+ context.withValue(context.value.copy(currentTypeAttrs = attrs)){f}
+ context.value.traits +=
+ TypesUtil.createTraitTypeDef(name, ImmutableList.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*)
+ val v = context.value
+ v.traits.size
+ }
+
+ def struct(name : String)(f : => Unit): Unit = {
+ val attrs = new ArrayBuffer[Attr]()
+ context.withValue(context.value.copy(currentTypeAttrs = attrs)){f}
+ context.value.structs +=
+ new StructTypeDefinition(name, attrs.map(_.getDef).toArray)
+ }
+
+ def enum(name : String, values : String*) : Unit = {
+ val enums = values.zipWithIndex.map{ case (v, i) =>
+ new EnumValue(v,i)
+ }
+ context.value.enums +=
+ TypesUtil.createEnumTypeDef(name, enums:_*)
+ }
+
+}
[41/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
Refactor packages and scripts to Atlas
(cherry picked from commit 414beba)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/30711973
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/30711973
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/30711973
Branch: refs/heads/master
Commit: 307119733894a5bb9b121b32f75f469c4c2ddf16
Parents: 2604810
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Mon Jun 8 17:41:01 2015 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Mon Jun 8 20:43:10 2015 -0700
----------------------------------------------------------------------
.../org/apache/atlas/falcon/FalconImporter.java | 144 ++
.../apache/atlas/falcon/FalconTypeSystem.java | 169 +++
.../apache/metadata/falcon/FalconImporter.java | 144 --
.../metadata/falcon/FalconTypeSystem.java | 176 ---
.../apache/atlas/falcon/FalconImporterTest.java | 108 ++
.../atlas/falcon/FalconTypeSystemTest.java | 35 +
.../metadata/falcon/FalconImporterTest.java | 109 --
.../metadata/falcon/FalconTypeSystemTest.java | 35 -
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 512 +++++++
.../org/apache/atlas/hive/hook/HiveHook.java | 339 +++++
.../hive/model/HiveDataModelGenerator.java | 482 +++++++
.../apache/atlas/hive/model/HiveDataTypes.java | 52 +
.../hive/bridge/HiveMetaStoreBridge.java | 512 -------
.../hadoop/metadata/hive/hook/HiveHook.java | 339 -----
.../hive/model/HiveDataModelGenerator.java | 482 -------
.../metadata/hive/model/HiveDataTypes.java | 52 -
.../atlas/hive/hook/BaseSSLAndKerberosTest.java | 142 ++
.../org/apache/atlas/hive/hook/HiveHookIT.java | 379 +++++
.../hook/NegativeSSLAndKerberosHiveHookIT.java | 162 +++
.../hive/hook/SSLAndKerberosHiveHookIT.java | 229 ++++
.../apache/atlas/hive/hook/SSLHiveHookIT.java | 224 +++
.../hive/hook/BaseSSLAndKerberosTest.java | 141 --
.../hadoop/metadata/hive/hook/HiveHookIT.java | 379 -----
.../hook/NegativeSSLAndKerberosHiveHookIT.java | 159 ---
.../hive/hook/SSLAndKerberosHiveHookIT.java | 223 ---
.../metadata/hive/hook/SSLHiveHookIT.java | 219 ---
.../org/apache/atlas/MetadataServiceClient.java | 368 +++++
.../apache/atlas/MetadataServiceException.java | 45 +
.../java/org/apache/atlas/PropertiesUtil.java | 58 +
.../atlas/security/SecureClientUtils.java | 195 +++
.../atlas/security/SecurityProperties.java | 36 +
.../hadoop/metadata/MetadataServiceClient.java | 368 -----
.../metadata/MetadataServiceException.java | 45 -
.../apache/hadoop/metadata/PropertiesUtil.java | 58 -
.../metadata/security/SecureClientUtils.java | 195 ---
.../metadata/security/SecurityProperties.java | 36 -
.../apache/atlas/security/BaseSecurityTest.java | 128 ++
.../metadata/security/BaseSecurityTest.java | 128 --
.../java/org/apache/atlas/GraphTransaction.java | 26 +
.../atlas/GraphTransactionInterceptor.java | 51 +
.../apache/atlas/RepositoryMetadataModule.java | 77 ++
.../atlas/discovery/DiscoveryException.java | 74 +
.../atlas/discovery/DiscoveryService.java | 52 +
.../atlas/discovery/HiveLineageService.java | 229 ++++
.../apache/atlas/discovery/LineageService.java | 67 +
.../apache/atlas/discovery/SearchIndexer.java | 29 +
.../graph/DefaultGraphPersistenceStrategy.java | 210 +++
.../graph/GraphBackedDiscoveryService.java | 219 +++
.../atlas/listener/EntityChangeListener.java | 54 +
.../atlas/listener/TypesChangeListener.java | 48 +
.../org/apache/atlas/repository/Constants.java | 84 ++
.../atlas/repository/DiscoverInstances.java | 76 ++
.../repository/EntityNotFoundException.java | 44 +
.../apache/atlas/repository/IRepository.java | 50 +
.../atlas/repository/MetadataRepository.java | 162 +++
.../atlas/repository/RepositoryException.java | 47 +
.../graph/GraphBackedMetadataRepository.java | 1292 ++++++++++++++++++
.../graph/GraphBackedSearchIndexer.java | 348 +++++
.../atlas/repository/graph/GraphHelper.java | 139 ++
.../atlas/repository/graph/GraphProvider.java | 28 +
.../repository/graph/TitanGraphProvider.java | 78 ++
.../repository/memory/AttributeStores.java | 655 +++++++++
.../atlas/repository/memory/ClassStore.java | 94 ++
.../memory/HierarchicalTypeStore.java | 209 +++
.../repository/memory/IAttributeStore.java | 49 +
.../atlas/repository/memory/MemRepository.java | 329 +++++
.../memory/ReplaceIdWithInstance.java | 143 ++
.../atlas/repository/memory/StructStore.java | 87 ++
.../atlas/repository/memory/TraitStore.java | 57 +
.../typestore/GraphBackedTypeStore.java | 329 +++++
.../atlas/repository/typestore/ITypeStore.java | 48 +
.../repository/typestore/StorageException.java | 35 +
.../atlas/services/DefaultMetadataService.java | 427 ++++++
.../apache/atlas/services/MetadataService.java | 125 ++
.../hadoop/metadata/GraphTransaction.java | 26 -
.../metadata/GraphTransactionInterceptor.java | 51 -
.../metadata/RepositoryMetadataModule.java | 78 --
.../metadata/discovery/DiscoveryException.java | 74 -
.../metadata/discovery/DiscoveryService.java | 54 -
.../metadata/discovery/HiveLineageService.java | 229 ----
.../metadata/discovery/LineageService.java | 67 -
.../metadata/discovery/SearchIndexer.java | 29 -
.../graph/DefaultGraphPersistenceStrategy.java | 203 ---
.../graph/GraphBackedDiscoveryService.java | 219 ---
.../metadata/listener/EntityChangeListener.java | 54 -
.../metadata/listener/TypesChangeListener.java | 48 -
.../hadoop/metadata/repository/Constants.java | 84 --
.../metadata/repository/DiscoverInstances.java | 76 --
.../repository/EntityNotFoundException.java | 46 -
.../hadoop/metadata/repository/IRepository.java | 50 -
.../metadata/repository/MetadataRepository.java | 162 ---
.../repository/RepositoryException.java | 47 -
.../graph/GraphBackedMetadataRepository.java | 1292 ------------------
.../graph/GraphBackedSearchIndexer.java | 351 -----
.../metadata/repository/graph/GraphHelper.java | 139 --
.../repository/graph/GraphProvider.java | 28 -
.../repository/graph/TitanGraphProvider.java | 79 --
.../repository/memory/AttributeStores.java | 655 ---------
.../metadata/repository/memory/ClassStore.java | 94 --
.../memory/HierarchicalTypeStore.java | 209 ---
.../repository/memory/IAttributeStore.java | 49 -
.../repository/memory/MemRepository.java | 329 -----
.../memory/ReplaceIdWithInstance.java | 143 --
.../metadata/repository/memory/StructStore.java | 87 --
.../metadata/repository/memory/TraitStore.java | 57 -
.../typestore/GraphBackedTypeStore.java | 330 -----
.../repository/typestore/ITypeStore.java | 48 -
.../repository/typestore/StorageException.java | 35 -
.../services/DefaultMetadataService.java | 427 ------
.../metadata/services/MetadataService.java | 125 --
.../org/apache/atlas/query/ClosureQuery.scala | 329 +++++
.../org/apache/atlas/query/Expressions.scala | 739 ++++++++++
.../query/GraphPersistenceStrategies.scala | 327 +++++
.../apache/atlas/query/GremlinEvaluator.scala | 139 ++
.../org/apache/atlas/query/GremlinQuery.scala | 348 +++++
.../org/apache/atlas/query/QueryParser.scala | 398 ++++++
.../org/apache/atlas/query/QueryProcessor.scala | 61 +
.../scala/org/apache/atlas/query/Resolver.scala | 157 +++
.../org/apache/atlas/query/TypeUtils.scala | 263 ++++
.../hadoop/metadata/query/ClosureQuery.scala | 330 -----
.../hadoop/metadata/query/Expressions.scala | 739 ----------
.../query/GraphPersistenceStrategies.scala | 327 -----
.../metadata/query/GremlinEvaluator.scala | 139 --
.../hadoop/metadata/query/GremlinQuery.scala | 349 -----
.../hadoop/metadata/query/QueryParser.scala | 398 ------
.../hadoop/metadata/query/QueryProcessor.scala | 61 -
.../apache/hadoop/metadata/query/Resolver.scala | 157 ---
.../hadoop/metadata/query/TypeUtils.scala | 263 ----
.../atlas/RepositoryServiceLoadingTest.java | 46 +
.../test/java/org/apache/atlas/TestUtils.java | 172 +++
.../GraphBackedDiscoveryServiceTest.java | 320 +++++
.../atlas/discovery/HiveLineageServiceTest.java | 593 ++++++++
.../org/apache/atlas/repository/BaseTest.java | 206 +++
.../GraphBackedMetadataRepositoryTest.java | 676 +++++++++
.../graph/GraphRepoMapperScaleTest.java | 272 ++++
.../atlas/repository/memory/ClassTest.java | 76 ++
.../atlas/repository/memory/EnumTest.java | 315 +++++
.../repository/memory/InstanceE2ETest.java | 167 +++
.../atlas/repository/memory/StorageTest.java | 127 ++
.../atlas/repository/memory/StructTest.java | 112 ++
.../atlas/repository/memory/TraitTest.java | 207 +++
.../typestore/GraphBackedTypeStoreTest.java | 128 ++
.../metadata/RepositoryServiceLoadingTest.java | 46 -
.../org/apache/hadoop/metadata/TestUtils.java | 172 ---
.../GraphBackedDiscoveryServiceTest.java | 320 -----
.../discovery/HiveLineageServiceTest.java | 590 --------
.../hadoop/metadata/repository/BaseTest.java | 206 ---
.../GraphBackedMetadataRepositoryTest.java | 676 ---------
.../graph/GraphRepoMapperScaleTest.java | 272 ----
.../metadata/repository/memory/ClassTest.java | 76 --
.../metadata/repository/memory/EnumTest.java | 315 -----
.../repository/memory/InstanceE2ETest.java | 164 ---
.../metadata/repository/memory/StorageTest.java | 127 --
.../metadata/repository/memory/StructTest.java | 112 --
.../metadata/repository/memory/TraitTest.java | 207 ---
.../typestore/GraphBackedTypeStoreTest.java | 128 --
.../org/apache/atlas/query/ExpressionTest.scala | 171 +++
.../org/apache/atlas/query/GremlinTest.scala | 153 +++
.../org/apache/atlas/query/GremlinTest2.scala | 135 ++
.../apache/atlas/query/HiveTitanSample.scala | 345 +++++
.../org/apache/atlas/query/LexerTest.scala | 48 +
.../apache/atlas/query/LineageQueryTest.scala | 515 +++++++
.../org/apache/atlas/query/ParserTest.scala | 101 ++
.../apache/atlas/query/QueryTestsUtils.scala | 169 +++
.../hadoop/metadata/query/ExpressionTest.scala | 171 ---
.../hadoop/metadata/query/GremlinTest.scala | 153 ---
.../hadoop/metadata/query/GremlinTest2.scala | 136 --
.../hadoop/metadata/query/HiveTitanSample.scala | 345 -----
.../hadoop/metadata/query/LexerTest.scala | 48 -
.../metadata/query/LineageQueryTest.scala | 515 -------
.../hadoop/metadata/query/ParserTest.scala | 101 --
.../hadoop/metadata/query/QueryTestsUtils.scala | 169 ---
src/bin/atlas_config.py | 285 ++++
src/bin/atlas_start.py | 81 ++
src/bin/atlas_stop.py | 59 +
src/bin/metadata_config.py | 285 ----
src/bin/metadata_start.py | 81 --
src/bin/metadata_stop.py | 59 -
.../atlas/tools/dsl/DynamicTypedStruct.scala | 38 +
.../org/apache/atlas/tools/dsl/package.scala | 99 ++
.../apache/atlas/tools/simpleserver/Main.scala | 59 +
.../tools/simpleserver/MetadataActor.scala | 121 ++
.../tools/simpleserver/RestInterface.scala | 126 ++
.../atlas/tools/thrift/ThriftParser.scala | 664 +++++++++
.../atlas/tools/thrift/ThriftTypesGen.scala | 316 +++++
.../metadata/tools/dsl/DynamicTypedStruct.scala | 38 -
.../hadoop/metadata/tools/dsl/package.scala | 102 --
.../metadata/tools/simpleserver/Main.scala | 59 -
.../tools/simpleserver/MetadataActor.scala | 121 --
.../tools/simpleserver/RestInterface.scala | 124 --
.../metadata/tools/thrift/ThriftParser.scala | 665 ---------
.../metadata/tools/thrift/ThriftTypesGen.scala | 316 -----
.../org/apache/atlas/tools/dsl/DSLTest.scala | 241 ++++
.../tools/hive/HiveMockMetadataService.scala | 84 ++
.../atlas/tools/thrift/ThriftLexerTest.scala | 525 +++++++
.../atlas/tools/thrift/ThriftParserTest.scala | 772 +++++++++++
.../hadoop/metadata/tools/dsl/DSLTest.scala | 242 ----
.../tools/hive/HiveMockMetadataService.scala | 84 --
.../metadata/tools/thrift/ThriftLexerTest.scala | 525 -------
.../tools/thrift/ThriftParserTest.scala | 772 -----------
.../org/apache/atlas/MetadataException.java | 45 +
.../java/org/apache/atlas/ParamChecker.java | 148 ++
.../atlas/classification/InterfaceAudience.java | 48 +
.../org/apache/atlas/typesystem/IInstance.java | 38 +
.../typesystem/IReferenceableInstance.java | 35 +
.../org/apache/atlas/typesystem/IStruct.java | 26 +
.../typesystem/ITypedReferenceableInstance.java | 22 +
.../apache/atlas/typesystem/ITypedStruct.java | 22 +
.../apache/atlas/typesystem/Referenceable.java | 93 ++
.../org/apache/atlas/typesystem/Struct.java | 63 +
.../persistence/DownCastStructInstance.java | 71 +
.../apache/atlas/typesystem/persistence/Id.java | 242 ++++
.../atlas/typesystem/persistence/MapIds.java | 71 +
.../persistence/ReferenceableInstance.java | 103 ++
.../typesystem/persistence/StructInstance.java | 757 ++++++++++
.../typesystem/types/AbstractDataType.java | 37 +
.../typesystem/types/AttributeDefinition.java | 93 ++
.../atlas/typesystem/types/AttributeInfo.java | 94 ++
.../atlas/typesystem/types/ClassType.java | 231 ++++
.../atlas/typesystem/types/DataTypes.java | 660 +++++++++
.../typesystem/types/DownCastFieldMapping.java | 53 +
.../apache/atlas/typesystem/types/EnumType.java | 90 ++
.../typesystem/types/EnumTypeDefinition.java | 54 +
.../atlas/typesystem/types/EnumValue.java | 57 +
.../atlas/typesystem/types/FieldMapping.java | 135 ++
.../typesystem/types/HierarchicalType.java | 499 +++++++
.../types/HierarchicalTypeDefinition.java | 76 ++
.../typesystem/types/IConstructableType.java | 34 +
.../atlas/typesystem/types/IDataType.java | 31 +
.../atlas/typesystem/types/Multiplicity.java | 90 ++
.../typesystem/types/ObjectGraphTraversal.java | 198 +++
.../typesystem/types/ObjectGraphWalker.java | 221 +++
.../atlas/typesystem/types/StructType.java | 204 +++
.../typesystem/types/StructTypeDefinition.java | 62 +
.../atlas/typesystem/types/TraitType.java | 77 ++
.../atlas/typesystem/types/TypeSystem.java | 673 +++++++++
.../atlas/typesystem/types/TypeUtils.java | 92 ++
.../typesystem/types/TypedStructHandler.java | 117 ++
.../types/ValueConversionException.java | 55 +
.../atlas/typesystem/types/package-info.java | 125 ++
.../atlas/typesystem/types/utils/TypesUtil.java | 85 ++
.../hadoop/metadata/MetadataException.java | 45 -
.../apache/hadoop/metadata/ParamChecker.java | 148 --
.../classification/InterfaceAudience.java | 48 -
.../hadoop/metadata/typesystem/IInstance.java | 38 -
.../typesystem/IReferenceableInstance.java | 35 -
.../hadoop/metadata/typesystem/IStruct.java | 26 -
.../typesystem/ITypedReferenceableInstance.java | 22 -
.../metadata/typesystem/ITypedStruct.java | 22 -
.../metadata/typesystem/Referenceable.java | 93 --
.../hadoop/metadata/typesystem/Struct.java | 63 -
.../persistence/DownCastStructInstance.java | 71 -
.../metadata/typesystem/persistence/Id.java | 242 ----
.../metadata/typesystem/persistence/MapIds.java | 71 -
.../persistence/ReferenceableInstance.java | 103 --
.../typesystem/persistence/StructInstance.java | 757 ----------
.../typesystem/types/AbstractDataType.java | 37 -
.../typesystem/types/AttributeDefinition.java | 93 --
.../typesystem/types/AttributeInfo.java | 94 --
.../metadata/typesystem/types/ClassType.java | 231 ----
.../metadata/typesystem/types/DataTypes.java | 660 ---------
.../typesystem/types/DownCastFieldMapping.java | 53 -
.../metadata/typesystem/types/EnumType.java | 90 --
.../typesystem/types/EnumTypeDefinition.java | 54 -
.../metadata/typesystem/types/EnumValue.java | 57 -
.../metadata/typesystem/types/FieldMapping.java | 135 --
.../typesystem/types/HierarchicalType.java | 500 -------
.../types/HierarchicalTypeDefinition.java | 76 --
.../typesystem/types/IConstructableType.java | 34 -
.../metadata/typesystem/types/IDataType.java | 31 -
.../metadata/typesystem/types/Multiplicity.java | 90 --
.../typesystem/types/ObjectGraphTraversal.java | 198 ---
.../typesystem/types/ObjectGraphWalker.java | 221 ---
.../metadata/typesystem/types/StructType.java | 204 ---
.../typesystem/types/StructTypeDefinition.java | 62 -
.../metadata/typesystem/types/TraitType.java | 77 --
.../metadata/typesystem/types/TypeSystem.java | 674 ---------
.../metadata/typesystem/types/TypeUtils.java | 92 --
.../typesystem/types/TypedStructHandler.java | 117 --
.../types/ValueConversionException.java | 55 -
.../metadata/typesystem/types/package-info.java | 125 --
.../typesystem/types/utils/TypesUtil.java | 85 --
.../org/apache/atlas/typesystem/TypesDef.scala | 62 +
.../typesystem/builders/InstanceBuilder.scala | 217 +++
.../typesystem/builders/TypesBuilder.scala | 188 +++
.../typesystem/json/InstanceSerialization.scala | 320 +++++
.../atlas/typesystem/json/Serialization.scala | 303 ++++
.../typesystem/json/TypesSerialization.scala | 244 ++++
.../hadoop/metadata/typesystem/TypesDef.scala | 65 -
.../typesystem/builders/InstanceBuilder.scala | 218 ---
.../typesystem/builders/TypesBuilder.scala | 188 ---
.../typesystem/json/InstanceSerialization.scala | 320 -----
.../typesystem/json/Serialization.scala | 303 ----
.../typesystem/json/TypesSerialization.scala | 244 ----
.../typesystem/json/SerializationJavaTest.java | 177 +++
.../apache/atlas/typesystem/types/BaseTest.java | 193 +++
.../atlas/typesystem/types/ClassTest.java | 72 +
.../apache/atlas/typesystem/types/EnumTest.java | 221 +++
.../atlas/typesystem/types/StructTest.java | 82 ++
.../atlas/typesystem/types/TraitTest.java | 224 +++
.../typesystem/types/TypeInheritanceTest.java | 256 ++++
.../atlas/typesystem/types/TypeSystemTest.java | 139 ++
.../atlas/typesystem/types/ValidationTest.java | 102 ++
.../typesystem/json/SerializationJavaTest.java | 177 ---
.../metadata/typesystem/types/BaseTest.java | 194 ---
.../metadata/typesystem/types/ClassTest.java | 72 -
.../metadata/typesystem/types/EnumTest.java | 221 ---
.../metadata/typesystem/types/StructTest.java | 82 --
.../metadata/typesystem/types/TraitTest.java | 224 ---
.../typesystem/types/TypeInheritanceTest.java | 256 ----
.../typesystem/types/TypeSystemTest.java | 139 --
.../typesystem/types/ValidationTest.java | 102 --
.../atlas/typesystem/builders/BuilderTest.scala | 80 ++
.../builders/InstanceBuilderTest.scala | 105 ++
.../typesystem/builders/MultiplicityTest.scala | 125 ++
.../typesystem/builders/TypesBuilderTest.scala | 35 +
.../typesystem/json/SerializationTest.scala | 242 ++++
.../json/TypesSerializationTest.scala | 300 ++++
.../typesystem/builders/BuilderTest.scala | 80 --
.../builders/InstanceBuilderTest.scala | 105 --
.../typesystem/builders/MultiplicityTest.scala | 125 --
.../typesystem/builders/TypesBuilderTest.scala | 36 -
.../typesystem/json/SerializationTest.scala | 242 ----
.../json/TypesSerializationTest.scala | 300 ----
webapp/src/main/java/org/apache/atlas/Main.java | 144 ++
.../org/apache/atlas/examples/QuickStart.java | 478 +++++++
.../atlas/util/CredentialProviderUtility.java | 149 ++
.../web/errors/LoggingExceptionMapper.java | 63 +
.../apache/atlas/web/filters/AuditFilter.java | 108 ++
.../filters/MetadataAuthenticationFilter.java | 104 ++
.../atlas/web/listeners/GuiceServletConfig.java | 112 ++
.../atlas/web/listeners/LoginProcessor.java | 160 +++
.../apache/atlas/web/params/AbstractParam.java | 138 ++
.../apache/atlas/web/params/BooleanParam.java | 48 +
.../apache/atlas/web/params/DateTimeParam.java | 38 +
.../atlas/web/resources/AdminResource.java | 104 ++
.../atlas/web/resources/EntityResource.java | 355 +++++
.../web/resources/HiveLineageResource.java | 178 +++
.../resources/MetadataDiscoveryResource.java | 322 +++++
.../web/resources/RexsterGraphResource.java | 408 ++++++
.../atlas/web/resources/TypesResource.java | 181 +++
.../atlas/web/service/EmbeddedServer.java | 87 ++
.../atlas/web/service/SecureEmbeddedServer.java | 116 ++
.../apache/atlas/web/util/DateTimeHelper.java | 123 ++
.../org/apache/atlas/web/util/Servlets.java | 152 +++
.../java/org/apache/hadoop/metadata/Main.java | 144 --
.../hadoop/metadata/examples/QuickStart.java | 478 -------
.../util/CredentialProviderUtility.java | 146 --
.../web/errors/LoggingExceptionMapper.java | 63 -
.../metadata/web/filters/AuditFilter.java | 108 --
.../filters/MetadataAuthenticationFilter.java | 104 --
.../web/listeners/GuiceServletConfig.java | 121 --
.../metadata/web/listeners/LoginProcessor.java | 161 ---
.../metadata/web/params/AbstractParam.java | 138 --
.../metadata/web/params/BooleanParam.java | 48 -
.../metadata/web/params/DateTimeParam.java | 38 -
.../metadata/web/resources/AdminResource.java | 104 --
.../metadata/web/resources/EntityResource.java | 355 -----
.../web/resources/HiveLineageResource.java | 173 ---
.../resources/MetadataDiscoveryResource.java | 322 -----
.../web/resources/RexsterGraphResource.java | 408 ------
.../metadata/web/resources/TypesResource.java | 185 ---
.../metadata/web/service/EmbeddedServer.java | 87 --
.../web/service/SecureEmbeddedServer.java | 109 --
.../metadata/web/util/DateTimeHelper.java | 123 --
.../hadoop/metadata/web/util/Servlets.java | 152 ---
.../atlas/CredentialProviderUtilityIT.java | 272 ++++
.../MetadataAuthenticationKerberosFilterIT.java | 179 +++
.../MetadataAuthenticationSimpleFilterIT.java | 94 ++
.../atlas/web/listeners/LoginProcessorIT.java | 104 ++
.../web/resources/AdminJerseyResourceIT.java | 69 +
.../atlas/web/resources/BaseResourceIT.java | 102 ++
.../web/resources/EntityJerseyResourceIT.java | 679 +++++++++
.../resources/HiveLineageJerseyResourceIT.java | 399 ++++++
.../MetadataDiscoveryJerseyResourceIT.java | 223 +++
.../resources/RexsterGraphJerseyResourceIT.java | 86 ++
.../web/resources/TypesJerseyResourceIT.java | 235 ++++
.../web/service/SecureEmbeddedServerIT.java | 61 +
.../web/service/SecureEmbeddedServerITBase.java | 224 +++
.../metadata/CredentialProviderUtilityIT.java | 272 ----
.../MetadataAuthenticationKerberosFilterIT.java | 175 ---
.../MetadataAuthenticationSimpleFilterIT.java | 94 --
.../web/listeners/LoginProcessorIT.java | 104 --
.../web/resources/AdminJerseyResourceIT.java | 69 -
.../metadata/web/resources/BaseResourceIT.java | 102 --
.../web/resources/EntityJerseyResourceIT.java | 679 ---------
.../resources/HiveLineageJerseyResourceIT.java | 399 ------
.../MetadataDiscoveryJerseyResourceIT.java | 223 ---
.../resources/RexsterGraphJerseyResourceIT.java | 86 --
.../web/resources/TypesJerseyResourceIT.java | 235 ----
.../web/service/SecureEmbeddedServerIT.java | 61 -
.../web/service/SecureEmbeddedServerITBase.java | 221 ---
392 files changed, 37072 insertions(+), 37068 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java
new file mode 100755
index 0000000..4ad1e3d
--- /dev/null
+++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.falcon;
+
+import com.google.inject.Inject;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.commons.lang.StringUtils;
+import org.apache.falcon.client.FalconCLIException;
+import org.apache.falcon.client.FalconClient;
+import org.apache.falcon.entity.v0.Entity;
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.cluster.Cluster;
+import org.apache.falcon.entity.v0.cluster.Interface;
+import org.apache.falcon.entity.v0.cluster.Location;
+import org.apache.falcon.entity.v0.cluster.Properties;
+import org.apache.falcon.entity.v0.cluster.Property;
+import org.apache.falcon.resource.EntityList;
+
+import javax.xml.bind.JAXBException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class FalconImporter {
+
+ private static final TypeSystem typeSystem = TypeSystem.getInstance();
+
+ private final FalconClient client;
+ private final MetadataRepository repository;
+
+ @Inject
+ public FalconImporter(FalconClient client, MetadataRepository repo) {
+ this.client = client;
+ this.repository = repo;
+ }
+
+ private Entity getEntity(FalconClient client, EntityType type, String name) throws FalconCLIException, JAXBException {
+ String entityStr = client.getDefinition(type.name(), name);
+ return (Entity) type.getUnmarshaller().unmarshal(new StringReader(entityStr));
+ }
+
+ public void importClusters() throws MetadataException {
+ try {
+ EntityList clusters = client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null);
+ for (EntityList.EntityElement element : clusters.getElements()) {
+ Cluster cluster = (Cluster) getEntity(client, EntityType.CLUSTER, element.name);
+
+ Referenceable clusterRef = new Referenceable(FalconTypeSystem.DefinedTypes.CLUSTER.name());
+ clusterRef.set("name", cluster.getName());
+
+ if (cluster.getACL() != null) {
+ Struct acl = new Struct(FalconTypeSystem.DefinedTypes.ACL.name());
+ acl.set("owner", cluster.getACL().getOwner());
+ acl.set("group", cluster.getACL().getGroup());
+ acl.set("permission", cluster.getACL().getPermission());
+ StructType aclType = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.ACL.name());
+ clusterRef.set("acl", aclType.convert(acl, Multiplicity.REQUIRED));
+ }
+
+ if (StringUtils.isNotEmpty(cluster.getTags())) {
+ String[] parts = cluster.getTags().split(",");
+ List<ITypedInstance> tags = new ArrayList<>();
+ for (String part : parts) {
+ TraitType tagType = typeSystem.getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name());
+ String[] kv = part.trim().split("=");
+ Struct tag = new Struct(FalconTypeSystem.DefinedTypes.TAG.name());
+ tag.set("name", kv[0]);
+ tag.set("value", kv[0]);
+ tags.add(tagType.convert(tag, Multiplicity.REQUIRED));
+ }
+ clusterRef.set("tags", tags);
+ }
+
+ if (cluster.getProperties() != null) {
+ clusterRef.set("properties", getMap(cluster.getProperties()));
+ }
+
+ if (cluster.getLocations() != null) {
+ List<ITypedInstance> locations = new ArrayList<>();
+ for (Location loc : cluster.getLocations().getLocations()) {
+ Struct location = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name());
+ EnumType locationType = typeSystem.getDataType(EnumType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION_TYPE.name());
+ location.set("type", locationType.fromValue(loc.getName().toUpperCase()));
+ location.set("path", loc.getPath());
+ StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name());
+ locations.add(type.convert(location, Multiplicity.REQUIRED));
+ }
+ clusterRef.set("locations", locations);
+ }
+
+ if (cluster.getInterfaces() != null) {
+ List<ITypedInstance> interfaces = new ArrayList<>();
+ for (Interface interfaceFld : cluster.getInterfaces().getInterfaces()) {
+ Struct interfaceStruct = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name());
+ interfaceStruct.set("type", interfaceFld.getType().name());
+ interfaceStruct.set("endpoint", interfaceFld.getEndpoint());
+ interfaceStruct.set("version", interfaceFld.getVersion());
+ StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name());
+ interfaces.add(type.convert(interfaceStruct, Multiplicity.REQUIRED));
+ }
+ clusterRef.set("interfaces", interfaces);
+ }
+ repository.createEntity(clusterRef);
+ }
+ } catch (Exception e) {
+ throw new MetadataException(e);
+ }
+ }
+
+ private Map<String, String> getMap(Properties properties) {
+ Map<String, String> map = new HashMap();
+ for (Property property : properties.getProperties()) {
+ map.put(property.getName().trim(), property.getValue().trim());
+ }
+ return map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java
new file mode 100755
index 0000000..6cbd970
--- /dev/null
+++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.falcon;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class FalconTypeSystem {
+ public static final Logger LOG = LoggerFactory.getLogger(FalconTypeSystem.class);
+ public static final TypeSystem TYPE_SYSTEM = TypeSystem.getInstance();
+ private static FalconTypeSystem INSTANCE;
+
+ private List<StructTypeDefinition> structTypeDefinitions = new ArrayList<>();
+ private List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions = new ArrayList<>();
+
+ private FalconTypeSystem() throws MetadataException {
+ HierarchicalTypeDefinition<ClassType> cluster = defineCluster();
+ //TODO define feed and process
+
+ TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions),
+ ImmutableList.of(cluster));
+ }
+
+ public static FalconTypeSystem getInstance() throws MetadataException {
+ if (INSTANCE == null) {
+ synchronized(LOG) {
+ if (INSTANCE == null) {
+ INSTANCE = new FalconTypeSystem();
+ }
+ }
+ }
+ return INSTANCE;
+ }
+
+ private HierarchicalTypeDefinition<ClassType> defineCluster() throws MetadataException {
+ defineACL();
+ defineClusterInterface();
+ defineClusterLocation();
+ defineTags();
+
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("acl", DefinedTypes.ACL.name(), Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("tags", DefinedTypes.TAG.name(), Multiplicity.COLLECTION, false, null),
+ new AttributeDefinition("locations", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.COLLECTION, false, null),
+ new AttributeDefinition("interfaces", DefinedTypes.CLUSTER_INTERFACE.name(), Multiplicity.COLLECTION, false, null),
+ new AttributeDefinition("properties", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),
+ };
+ HierarchicalTypeDefinition<ClassType> cluster =
+ new HierarchicalTypeDefinition<>(ClassType.class, DefinedTypes.CLUSTER.name(), ImmutableList.<String>of(), attributeDefinitions);
+ LOG.debug("Created definition for " + DefinedTypes.CLUSTER.name());
+ return cluster;
+ }
+
+ private HierarchicalTypeDefinition<TraitType> defineTags() {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("value", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null)
+ };
+
+ HierarchicalTypeDefinition<TraitType> traitType = new HierarchicalTypeDefinition<>(TraitType.class, DefinedTypes.TAG.name(), ImmutableList.<String>of(), attributeDefinitions);
+ LOG.debug("Created definition for " + DefinedTypes.TAG.name());
+ traitTypeDefinitions.add(traitType);
+ return traitType;
+ }
+
+ private StructTypeDefinition defineClusterLocation() throws MetadataException {
+ EnumValue values[] = {
+ new EnumValue("WORKING", 1),
+ new EnumValue("STAGING", 2),
+ new EnumValue("TEMP", 3),
+ };
+
+ LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION_TYPE.name());
+ EnumTypeDefinition locationType = new EnumTypeDefinition(DefinedTypes.CLUSTER_LOCATION_TYPE.name(), values);
+ TYPE_SYSTEM.defineEnumType(locationType);
+
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("type", DefinedTypes.CLUSTER_LOCATION_TYPE.name(), Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("path", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ };
+ LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION.name());
+ StructTypeDefinition location = new StructTypeDefinition(DefinedTypes.CLUSTER_LOCATION.name(), attributeDefinitions);
+ structTypeDefinitions.add(location);
+ return location;
+ }
+
+ private StructTypeDefinition defineClusterInterface() throws MetadataException {
+ EnumValue values[] = {
+ new EnumValue("READONLY", 1),
+ new EnumValue("WRITE", 2),
+ new EnumValue("EXECUTE", 3),
+ new EnumValue("WORKFLOW", 4),
+ new EnumValue("MESSAGING", 5),
+ new EnumValue("REGISTRY", 6),
+ };
+
+ LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE_TYPE.name());
+ EnumTypeDefinition interfaceType = new EnumTypeDefinition(DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), values);
+ TYPE_SYSTEM.defineEnumType(interfaceType);
+
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("type", DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("endpoint", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("version", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ };
+ LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE.name());
+ StructTypeDefinition interfaceEntity = new StructTypeDefinition(DefinedTypes.CLUSTER_INTERFACE.name(), attributeDefinitions);
+ structTypeDefinitions.add(interfaceEntity);
+ return interfaceEntity;
+ }
+
+ public static enum DefinedTypes {
+ ACL,
+ TAG,
+
+ CLUSTER,
+ CLUSTER_INTERFACE,
+ CLUSTER_INTERFACE_TYPE,
+ CLUSTER_LOCATION,
+ CLUSTER_LOCATION_TYPE
+ }
+
+ private StructTypeDefinition defineACL() {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("group", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("permission", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+ LOG.debug("Created definition for " + DefinedTypes.ACL.name());
+ StructTypeDefinition acl = new StructTypeDefinition(DefinedTypes.ACL.name(), attributeDefinitions);
+ structTypeDefinitions.add(acl);
+ return acl;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java b/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java
deleted file mode 100755
index 1d01d75..0000000
--- a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconImporter.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metadata.falcon;
-
-import com.google.inject.Inject;
-import org.apache.falcon.client.FalconCLIException;
-import org.apache.falcon.client.FalconClient;
-import org.apache.falcon.entity.v0.Entity;
-import org.apache.falcon.entity.v0.EntityType;
-import org.apache.falcon.entity.v0.cluster.Cluster;
-import org.apache.falcon.entity.v0.cluster.Interface;
-import org.apache.falcon.entity.v0.cluster.Location;
-import org.apache.falcon.entity.v0.cluster.Properties;
-import org.apache.falcon.entity.v0.cluster.Property;
-import org.apache.falcon.resource.EntityList;
-import org.apache.hadoop.metadata.ITypedInstance;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.Referenceable;
-import org.apache.hadoop.metadata.Struct;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.apache.hadoop.metadata.typesystem.types.EnumType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.parboiled.common.StringUtils;
-
-import javax.xml.bind.JAXBException;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class FalconImporter {
-
- private static final TypeSystem typeSystem = TypeSystem.getInstance();
-
- private final FalconClient client;
- private final MetadataRepository repository;
-
- @Inject
- public FalconImporter(FalconClient client, MetadataRepository repo) {
- this.client = client;
- this.repository = repo;
- }
-
- private Entity getEntity(FalconClient client, EntityType type, String name) throws FalconCLIException, JAXBException {
- String entityStr = client.getDefinition(type.name(), name);
- return (Entity) type.getUnmarshaller().unmarshal(new StringReader(entityStr));
- }
-
- public void importClusters() throws MetadataException {
- try {
- EntityList clusters = client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null);
- for (EntityList.EntityElement element : clusters.getElements()) {
- Cluster cluster = (Cluster) getEntity(client, EntityType.CLUSTER, element.name);
-
- Referenceable clusterRef = new Referenceable(FalconTypeSystem.DefinedTypes.CLUSTER.name());
- clusterRef.set("name", cluster.getName());
-
- if (cluster.getACL() != null) {
- Struct acl = new Struct(FalconTypeSystem.DefinedTypes.ACL.name());
- acl.set("owner", cluster.getACL().getOwner());
- acl.set("group", cluster.getACL().getGroup());
- acl.set("permission", cluster.getACL().getPermission());
- StructType aclType = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.ACL.name());
- clusterRef.set("acl", aclType.convert(acl, Multiplicity.REQUIRED));
- }
-
- if (StringUtils.isNotEmpty(cluster.getTags())) {
- String[] parts = cluster.getTags().split(",");
- List<ITypedInstance> tags = new ArrayList<>();
- for (String part : parts) {
- TraitType tagType = typeSystem.getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name());
- String[] kv = part.trim().split("=");
- Struct tag = new Struct(FalconTypeSystem.DefinedTypes.TAG.name());
- tag.set("name", kv[0]);
- tag.set("value", kv[0]);
- tags.add(tagType.convert(tag, Multiplicity.REQUIRED));
- }
- clusterRef.set("tags", tags);
- }
-
- if (cluster.getProperties() != null) {
- clusterRef.set("properties", getMap(cluster.getProperties()));
- }
-
- if (cluster.getLocations() != null) {
- List<ITypedInstance> locations = new ArrayList<>();
- for (Location loc : cluster.getLocations().getLocations()) {
- Struct location = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name());
- EnumType locationType = typeSystem.getDataType(EnumType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION_TYPE.name());
- location.set("type", locationType.fromValue(loc.getName().toUpperCase()));
- location.set("path", loc.getPath());
- StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name());
- locations.add(type.convert(location, Multiplicity.REQUIRED));
- }
- clusterRef.set("locations", locations);
- }
-
- if (cluster.getInterfaces() != null) {
- List<ITypedInstance> interfaces = new ArrayList<>();
- for (Interface interfaceFld : cluster.getInterfaces().getInterfaces()) {
- Struct interfaceStruct = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name());
- interfaceStruct.set("type", interfaceFld.getType().name());
- interfaceStruct.set("endpoint", interfaceFld.getEndpoint());
- interfaceStruct.set("version", interfaceFld.getVersion());
- StructType type = typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name());
- interfaces.add(type.convert(interfaceStruct, Multiplicity.REQUIRED));
- }
- clusterRef.set("interfaces", interfaces);
- }
- repository.createEntity(clusterRef, clusterRef.getTypeName());
- }
- } catch (Exception e) {
- throw new MetadataException(e);
- }
- }
-
- private Map<String, String> getMap(Properties properties) {
- Map<String, String> map = new HashMap();
- for (Property property : properties.getProperties()) {
- map.put(property.getName().trim(), property.getValue().trim());
- }
- return map;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java b/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java
deleted file mode 100755
index 0388b84..0000000
--- a/addons/falcon-bridge/src/main/java/org/apache/metadata/falcon/FalconTypeSystem.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metadata.falcon;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.types.AttributeDefinition;
-import org.apache.hadoop.metadata.types.ClassType;
-import org.apache.hadoop.metadata.types.DataTypes;
-import org.apache.hadoop.metadata.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.types.EnumValue;
-import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.types.Multiplicity;
-import org.apache.hadoop.metadata.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.types.TraitType;
-import org.apache.hadoop.metadata.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class FalconTypeSystem {
- public static final Logger LOG = LoggerFactory.getLogger(FalconTypeSystem.class);
- public static final TypeSystem TYPE_SYSTEM = TypeSystem.getInstance();
-
- private List<StructTypeDefinition> structTypeDefinitions = new ArrayList<>();
- private List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions = new ArrayList<>();
-
- private FalconTypeSystem() throws MetadataException {
- HierarchicalTypeDefinition<ClassType> cluster = defineCluster();
- //TODO define feed and process
-
- TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions),
- ImmutableList.of(cluster));
- }
-
- public static FalconTypeSystem getInstance() throws MetadataException {
- if (INSTANCE == null) {
- synchronized(LOG) {
- if (INSTANCE == null) {
- INSTANCE = new FalconTypeSystem();
- }
- }
- }
- return INSTANCE;
- }
-
- private FalconTypeSystem() throws MetadataException {
- HierarchicalTypeDefinition<ClassType> cluster = defineCluster();
- //TODO define feed and process
-
- TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions),
- ImmutableList.of(cluster));
- }
-
- private HierarchicalTypeDefinition<ClassType> defineCluster() throws MetadataException {
- defineACL();
- defineClusterInterface();
- defineClusterLocation();
- defineTags();
-
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("acl", DefinedTypes.ACL.name(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("tags", DefinedTypes.TAG.name(), Multiplicity.COLLECTION, false, null),
- new AttributeDefinition("locations", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.COLLECTION, false, null),
- new AttributeDefinition("interfaces", DefinedTypes.CLUSTER_INTERFACE.name(), Multiplicity.COLLECTION, false, null),
- new AttributeDefinition("properties", TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),
- };
- HierarchicalTypeDefinition<ClassType> cluster =
- new HierarchicalTypeDefinition<>(ClassType.class, DefinedTypes.CLUSTER.name(), ImmutableList.<String>of(), attributeDefinitions);
- LOG.debug("Created definition for " + DefinedTypes.CLUSTER.name());
- return cluster;
- }
-
- private HierarchicalTypeDefinition<TraitType> defineTags() {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("value", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null)
- };
-
- HierarchicalTypeDefinition<TraitType> traitType = new HierarchicalTypeDefinition<>(TraitType.class, DefinedTypes.TAG.name(), ImmutableList.<String>of(), attributeDefinitions);
- LOG.debug("Created definition for " + DefinedTypes.TAG.name());
- traitTypeDefinitions.add(traitType);
- return traitType;
- }
-
- private StructTypeDefinition defineClusterLocation() throws MetadataException {
- EnumValue values[] = {
- new EnumValue("WORKING", 1),
- new EnumValue("STAGING", 2),
- new EnumValue("TEMP", 3),
- };
-
- LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION_TYPE.name());
- EnumTypeDefinition locationType = new EnumTypeDefinition(DefinedTypes.CLUSTER_LOCATION_TYPE.name(), values);
- TYPE_SYSTEM.defineEnumType(locationType);
-
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("type", DefinedTypes.CLUSTER_LOCATION_TYPE.name(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("path", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- };
- LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION.name());
- StructTypeDefinition location = new StructTypeDefinition(DefinedTypes.CLUSTER_LOCATION.name(), attributeDefinitions);
- structTypeDefinitions.add(location);
- return location;
- }
-
- private StructTypeDefinition defineClusterInterface() throws MetadataException {
- EnumValue values[] = {
- new EnumValue("READONLY", 1),
- new EnumValue("WRITE", 2),
- new EnumValue("EXECUTE", 3),
- new EnumValue("WORKFLOW", 4),
- new EnumValue("MESSAGING", 5),
- new EnumValue("REGISTRY", 6),
- };
-
- LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE_TYPE.name());
- EnumTypeDefinition interfaceType = new EnumTypeDefinition(DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), values);
- TYPE_SYSTEM.defineEnumType(interfaceType);
-
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("type", DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("endpoint", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("version", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- };
- LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE.name());
- StructTypeDefinition interfaceEntity = new StructTypeDefinition(DefinedTypes.CLUSTER_INTERFACE.name(), attributeDefinitions);
- structTypeDefinitions.add(interfaceEntity);
- return interfaceEntity;
- }
-
- public static enum DefinedTypes {
- ACL,
- TAG,
-
- CLUSTER,
- CLUSTER_INTERFACE,
- CLUSTER_INTERFACE_TYPE,
- CLUSTER_LOCATION,
- CLUSTER_LOCATION_TYPE
- }
-
- private StructTypeDefinition defineACL() {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("group", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("permission", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
- LOG.debug("Created definition for " + DefinedTypes.ACL.name());
- StructTypeDefinition acl = new StructTypeDefinition(DefinedTypes.ACL.name(), attributeDefinitions);
- structTypeDefinitions.add(acl);
- return acl;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java
new file mode 100755
index 0000000..7562b89
--- /dev/null
+++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.falcon;
+
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.falcon.client.FalconClient;
+import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.cluster.Cluster;
+import org.apache.falcon.entity.v0.cluster.Interface;
+import org.apache.falcon.entity.v0.cluster.Interfaces;
+import org.apache.falcon.entity.v0.cluster.Interfacetype;
+import org.apache.falcon.entity.v0.cluster.Location;
+import org.apache.falcon.entity.v0.cluster.Locations;
+import org.apache.falcon.resource.EntityList;
+import org.testng.annotations.Test;
+
+import java.io.StringWriter;
+import java.util.UUID;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class FalconImporterTest {
+ @Test
+ public void testImport() throws Exception {
+ MetadataRepository repo = mock(MetadataRepository.class);
+ FalconClient client = mock(FalconClient.class);
+ FalconTypeSystem.getInstance();
+
+ FalconImporter importer = new FalconImporter(client, repo);
+ when(client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null,
+ null)).thenReturn(getEntityList());
+ //TODO Set other fields in cluster
+ when(client.getDefinition(anyString(), anyString())).thenReturn(getCluster());
+ when(repo.createEntity(any(IReferenceableInstance.class), anyString())).thenReturn(UUID.randomUUID().toString());
+
+ importer.importClusters();
+ }
+
+ public EntityList getEntityList() {
+ EntityList.EntityElement[] entities = new EntityList.EntityElement[2];
+ entities[0] = new EntityList.EntityElement();
+ entities[0].name = "c1";
+ entities[1] = new EntityList.EntityElement();
+ entities[1].name = "c2";
+ return new EntityList(entities);
+ }
+
+ private Interface getInterface(Interfacetype type, String endpoint) {
+ Interface clusterInterface = new Interface();
+ clusterInterface.setEndpoint(endpoint);
+ clusterInterface.setType(type);
+ clusterInterface.setVersion("2.2");
+ return clusterInterface;
+ }
+
+ public String getCluster() throws Exception {
+ Cluster cluster = new Cluster();
+ cluster.setName(RandomStringUtils.randomAlphabetic(10));
+ cluster.setColo(RandomStringUtils.randomAlphabetic(5));
+
+ cluster.setTags("owner=xyz,team=abc");
+ Interfaces interfaces = new Interfaces();
+ Interface clusterInterface = new Interface();
+ clusterInterface.setEndpoint("hdfs://localhost:8030");
+ clusterInterface.setType(Interfacetype.WRITE);
+ clusterInterface.setVersion("2.2");
+ interfaces.getInterfaces().add(getInterface(Interfacetype.WRITE, "hdfs://localhost:8030"));
+ interfaces.getInterfaces().add(getInterface(Interfacetype.READONLY, "hdfs://localhost:8030"));
+ interfaces.getInterfaces().add(getInterface(Interfacetype.EXECUTE, "http://localhost:8040"));
+ cluster.setInterfaces(interfaces);
+
+ Locations locations = new Locations();
+ locations.getLocations().add(getLocation());
+ cluster.setLocations(locations);
+
+ StringWriter writer = new StringWriter();
+ EntityType.CLUSTER.getMarshaller().marshal(cluster, writer);
+ return writer.toString();
+ }
+
+ public Location getLocation() {
+ Location location = new Location();
+ location.setName("staging");
+ location.setPath("/staging");
+ return location;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java
new file mode 100755
index 0000000..cc2b65f
--- /dev/null
+++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.falcon;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.junit.Assert;
+import org.testng.annotations.Test;
+
+public class FalconTypeSystemTest {
+ @Test
+ public void testTypeSystem() throws MetadataException {
+ FalconTypeSystem.getInstance();
+ Assert.assertNotNull(TypeSystem.getInstance().getDataType(ClassType.class, FalconTypeSystem.DefinedTypes.CLUSTER.name()));
+ Assert.assertNotNull(TypeSystem.getInstance().getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java b/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java
deleted file mode 100755
index 1104b70..0000000
--- a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconImporterTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metadata.falcon;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.falcon.client.FalconCLIException;
-import org.apache.falcon.client.FalconClient;
-import org.apache.falcon.entity.v0.EntityType;
-import org.apache.falcon.entity.v0.cluster.Cluster;
-import org.apache.falcon.entity.v0.cluster.Interface;
-import org.apache.falcon.entity.v0.cluster.Interfaces;
-import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.cluster.Location;
-import org.apache.falcon.entity.v0.cluster.Locations;
-import org.apache.falcon.resource.EntityList;
-import org.apache.hadoop.metadata.IReferenceableInstance;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.testng.annotations.Test;
-
-import java.io.StringWriter;
-import java.util.UUID;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class FalconImporterTest {
- @Test
- public void testImport() throws Exception {
- MetadataRepository repo = mock(MetadataRepository.class);
- FalconClient client = mock(FalconClient.class);
- FalconTypeSystem.getInstance();
-
- FalconImporter importer = new FalconImporter(client, repo);
- when(client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null)).thenReturn(getEntityList());
- //TODO Set other fields in cluster
- when(client.getDefinition(anyString(), anyString())).thenReturn(getCluster());
- when(repo.createEntity(any(IReferenceableInstance.class), anyString())).thenReturn(UUID.randomUUID().toString());
-
- importer.importClusters();
- }
-
- public EntityList getEntityList() {
- EntityList.EntityElement[] entities = new EntityList.EntityElement[2];
- entities[0] = new EntityList.EntityElement();
- entities[0].name = "c1";
- entities[1] = new EntityList.EntityElement();
- entities[1].name = "c2";
- return new EntityList(entities);
- }
-
- private Interface getInterface(Interfacetype type, String endpoint) {
- Interface clusterInterface = new Interface();
- clusterInterface.setEndpoint(endpoint);
- clusterInterface.setType(type);
- clusterInterface.setVersion("2.2");
- return clusterInterface;
- }
-
- public String getCluster() throws Exception {
- Cluster cluster = new Cluster();
- cluster.setName(RandomStringUtils.randomAlphabetic(10));
- cluster.setColo(RandomStringUtils.randomAlphabetic(5));
-
- cluster.setTags("owner=xyz,team=abc");
- Interfaces interfaces = new Interfaces();
- Interface clusterInterface = new Interface();
- clusterInterface.setEndpoint("hdfs://localhost:8030");
- clusterInterface.setType(Interfacetype.WRITE);
- clusterInterface.setVersion("2.2");
- interfaces.getInterfaces().add(getInterface(Interfacetype.WRITE, "hdfs://localhost:8030"));
- interfaces.getInterfaces().add(getInterface(Interfacetype.READONLY, "hdfs://localhost:8030"));
- interfaces.getInterfaces().add(getInterface(Interfacetype.EXECUTE, "http://localhost:8040"));
- cluster.setInterfaces(interfaces);
-
- Locations locations = new Locations();
- locations.getLocations().add(getLocation());
- cluster.setLocations(locations);
-
- StringWriter writer = new StringWriter();
- EntityType.CLUSTER.getMarshaller().marshal(cluster, writer);
- return writer.toString();
- }
-
- public Location getLocation() {
- Location location = new Location();
- location.setName("staging");
- location.setPath("/staging");
- return location;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java b/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java
deleted file mode 100755
index a067249..0000000
--- a/addons/falcon-bridge/src/test/java/org/apache/metadata/falcon/FalconTypeSystemTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metadata.falcon;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.types.ClassType;
-import org.apache.hadoop.metadata.types.TraitType;
-import org.apache.hadoop.metadata.types.TypeSystem;
-import org.junit.Assert;
-import org.testng.annotations.Test;
-
-public class FalconTypeSystemTest {
- @Test
- public void testTypeSystem() throws MetadataException {
- FalconTypeSystem.getInstance();
- Assert.assertNotNull(TypeSystem.getInstance().getDataType(ClassType.class, FalconTypeSystem.DefinedTypes.CLUSTER.name()));
- Assert.assertNotNull(TypeSystem.getInstance().getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name()));
- }
-}
[28/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala
deleted file mode 100755
index aec424c..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala
+++ /dev/null
@@ -1,739 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import org.apache.hadoop.metadata.MetadataException
-import org.apache.hadoop.metadata.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
-import org.apache.hadoop.metadata.typesystem.types._
-
-object Expressions {
-
- import TypeUtils._
-
- class ExpressionException(val e: Expression, message: String, cause: Throwable, enableSuppression: Boolean,
- writableStackTrace: Boolean)
- extends MetadataException(message, cause, enableSuppression, writableStackTrace) {
-
- def this(e: Expression, message: String) {
- this(e, message, null, false, false)
- }
-
- def this(e: Expression, message: String, cause: Throwable) {
- this(e, message, cause, false, false)
- }
-
- def this(e: Expression, cause: Throwable) {
- this(e, null, cause, false, false)
- }
-
- override def getMessage: String = {
- val eString = e.toString
- s"${super.getMessage}, expression:${if (eString contains "\n") "\n" else " "}$e"
- }
-
- }
-
- class UnresolvedException(expr: Expression, function: String) extends
- ExpressionException(expr, s"Unresolved $function")
-
- def attachExpression[A](e: Expression, msg: String = "")(f: => A): A = {
- try f catch {
- case eex: ExpressionException => throw eex
- case ex: Exception => throw new ExpressionException(e, msg, ex)
- }
- }
-
- trait Expression {
- self: Product =>
-
- def children: Seq[Expression]
-
- /**
- * Returns `true` if the schema for this expression and all its children have been resolved.
- * The default logic is that an Expression is resolve if all its children are resolved.
- */
- lazy val resolved: Boolean = childrenResolved
-
- /**
- * Returns the output [[IDataType[_]] of this expression. Expressions that are unresolved will
- * throw if this method is invoked.
- */
- def dataType: IDataType[_]
-
- /**
- * Returns true if all the children have been resolved.
- */
- def childrenResolved = !children.exists(!_.resolved)
-
-
- /**
- * the aliases that are present in this Expression Tree
- */
- def namedExpressions: Map[String, Expression] = Map()
-
- def fastEquals(other: Expression): Boolean = {
- this.eq(other) || this == other
- }
-
- def makeCopy(newArgs: Array[AnyRef]): this.type = attachExpression(this, "makeCopy") {
- try {
- val defaultCtor = getClass.getConstructors.find(_.getParameterTypes.size != 0).head
- defaultCtor.newInstance(newArgs: _*).asInstanceOf[this.type]
- } catch {
- case e: java.lang.IllegalArgumentException =>
- throw new ExpressionException(
- this, s"Failed to copy node. Reason: ${e.getMessage}.")
- }
- }
-
- def transformChildrenDown(rule: PartialFunction[Expression, Expression]): this.type = {
- var changed = false
- val newArgs = productIterator.map {
- case arg: Expression if children contains arg =>
- val newChild = arg.asInstanceOf[Expression].transformDown(rule)
- if (!(newChild fastEquals arg)) {
- changed = true
- newChild
- } else {
- arg
- }
- case Some(arg: Expression) if children contains arg =>
- val newChild = arg.asInstanceOf[Expression].transformDown(rule)
- if (!(newChild fastEquals arg)) {
- changed = true
- Some(newChild)
- } else {
- Some(arg)
- }
- case m: Map[_, _] => m
- case args: Traversable[_] => args.map {
- case arg: Expression if children contains arg =>
- val newChild = arg.asInstanceOf[Expression].transformDown(rule)
- if (!(newChild fastEquals arg)) {
- changed = true
- newChild
- } else {
- arg
- }
- case other => other
- }
- case nonChild: AnyRef => nonChild
- case null => null
- }.toArray
- if (changed) makeCopy(newArgs) else this
- }
-
- def transformDown(rule: PartialFunction[Expression, Expression]): Expression = {
- val afterRule = rule.applyOrElse(this, identity[Expression])
- // Check if unchanged and then possibly return old copy to avoid gc churn.
- if (this fastEquals afterRule) {
- transformChildrenDown(rule)
- } else {
- afterRule.transformChildrenDown(rule)
- }
- }
-
- def traverseChildren(traverseFunc: (Expression, PartialFunction[Expression, Unit]) => Unit)
- (rule: PartialFunction[Expression, Unit]): Unit = {
- productIterator.foreach {
- case arg: Expression if children contains arg =>
- traverseFunc(arg.asInstanceOf[Expression], rule)
- case Some(arg: Expression) if children contains arg =>
- traverseFunc(arg.asInstanceOf[Expression], rule)
- case m: Map[_, _] => m
- case args: Traversable[_] => args.map {
- case arg: Expression if children contains arg =>
- traverseFunc(arg.asInstanceOf[Expression], rule)
- case other => other
- }
- case nonChild: AnyRef => nonChild
- case null => null
- }
- }
-
- def traverseChildrenDown = traverseChildren(_traverseDown) _
-
- private def _traverseDown(e: Expression, rule: PartialFunction[Expression, Unit]): Unit = {
- if (rule.isDefinedAt(e)) {
- rule.apply(e)
- }
- e.traverseChildrenDown(rule)
- }
-
- def traverseDown(rule: PartialFunction[Expression, Unit]): Unit = {
- _traverseDown(this, rule)
- }
-
- def traverseChildrenUp = traverseChildren(_traverseUp) _
-
- private def _traverseUp(e: Expression, rule: PartialFunction[Expression, Unit]): Unit = {
- e.traverseChildrenUp(rule)
- if (rule.isDefinedAt(e)) {
- rule.apply(e)
- }
- }
-
- def traverseUp(rule: PartialFunction[Expression, Unit]): Unit = {
- _traverseUp(this, rule)
- }
-
- def transformUp(rule: PartialFunction[Expression, Expression]): Expression = {
- val afterRuleOnChildren = transformChildrenUp(rule);
- if (this fastEquals afterRuleOnChildren) {
- rule.applyOrElse(this, identity[Expression])
- } else {
- rule.applyOrElse(afterRuleOnChildren, identity[Expression])
- }
- }
-
- def transformChildrenUp(rule: PartialFunction[Expression, Expression]): this.type = {
- var changed = false
- val newArgs = productIterator.map {
- case arg: Expression if children contains arg =>
- val newChild = arg.asInstanceOf[Expression].transformUp(rule)
- if (!(newChild fastEquals arg)) {
- changed = true
- newChild
- } else {
- arg
- }
- case Some(arg: Expression) if children contains arg =>
- val newChild = arg.asInstanceOf[Expression].transformUp(rule)
- if (!(newChild fastEquals arg)) {
- changed = true
- Some(newChild)
- } else {
- Some(arg)
- }
- case m: Map[_, _] => m
- case args: Traversable[_] => args.map {
- case arg: Expression if children contains arg =>
- val newChild = arg.asInstanceOf[Expression].transformUp(rule)
- if (!(newChild fastEquals arg)) {
- changed = true
- newChild
- } else {
- arg
- }
- case other => other
- }
- case nonChild: AnyRef => nonChild
- case null => null
- }.toArray
- if (changed) makeCopy(newArgs) else this
- }
-
- /*
- * treeString methods
- */
- def nodeName = getClass.getSimpleName
-
- def argString: String = productIterator.flatMap {
- case e: Expression if children contains e => Nil
- case e: Expression if e.toString contains "\n" => s"(${e.simpleString})" :: Nil
- case seq: Seq[_] => seq.mkString("[", ",", "]") :: Nil
- case set: Set[_] => set.mkString("{", ",", "}") :: Nil
- case f: IDataType[_] => f.getName :: Nil
- case other => other :: Nil
- }.mkString(", ")
-
- /** String representation of this node without any children */
- def simpleString = s"$nodeName $argString"
-
- protected def generateTreeString(depth: Int, builder: StringBuilder): StringBuilder = {
- builder.append(" " * depth)
- builder.append(simpleString)
- builder.append("\n")
- children.foreach(_.generateTreeString(depth + 1, builder))
- builder
- }
-
- def treeString = generateTreeString(0, new StringBuilder).toString
-
- /*
- * Fluent API methods
- */
- def field(fieldName: String) = new UnresolvedFieldExpression(this, fieldName)
-
- def join(fieldName: String) = field(fieldName)
-
- def `.`(fieldName: String) = field(fieldName)
-
- def as(alias: String) = new AliasExpression(this, alias)
-
- def arith(op: String)(rightExpr: Expression) = new ArithmeticExpression(op, this, rightExpr)
-
- def + = arith("+") _
-
- def - = arith("-") _
-
- def * = arith("*") _
-
- def / = arith("/") _
-
- def % = arith("%") _
-
- def isTrait(name: String) = new isTraitUnaryExpression(name, this)
-
- def hasField(name: String) = new hasFieldUnaryExpression(name, this)
-
- def compareOp(op: String)(rightExpr: Expression) = new ComparisonExpression(op, this, rightExpr)
-
- def `=` = compareOp("=") _
-
- def `!=` = compareOp("!=") _
-
- def `>` = compareOp(">") _
-
- def `>=` = compareOp(">=") _
-
- def `<` = compareOp("<") _
-
- def `<=` = compareOp("=") _
-
- def logicalOp(op: String)(rightExpr: Expression) = new LogicalExpression(op, List(this, rightExpr))
-
- def and = logicalOp("and") _
-
- def or = logicalOp("or") _
-
- def where(condExpr: Expression) = new FilterExpression(this, condExpr)
-
- def select(selectList: Expression*) = new SelectExpression(this, selectList.toList)
-
- def loop(loopingExpr: Expression) = new LoopExpression(this, loopingExpr, None)
-
- def loop(loopingExpr: Expression, times: Literal[Integer]) =
- new LoopExpression(this, loopingExpr, Some(times))
-
- def traitInstance() = new TraitInstanceExpression(this)
- def instance() = new InstanceExpression(this)
-
- def path() = new PathExpression(this)
- }
-
- trait BinaryNode {
- self: Expression =>
- def left: Expression
-
- def right: Expression
-
- def children = Seq(left, right)
-
- override def namedExpressions = left.namedExpressions ++ right.namedExpressions
- }
-
- trait LeafNode {
- def children = Nil
- }
-
- trait UnaryNode {
- self: Expression =>
- def child: Expression
-
- override def namedExpressions = child.namedExpressions
-
- def children = child :: Nil
- }
-
- abstract class BinaryExpression extends Expression with BinaryNode {
- self: Product =>
- def symbol: String
-
- override def toString = s"($left $symbol $right)"
- }
-
- case class ClassExpression(clsName: String) extends Expression with LeafNode {
- val dataType = typSystem.getDataType(classOf[ClassType], clsName)
-
- override def toString = clsName
- }
-
- def _class(name: String): Expression = new ClassExpression(name)
-
- case class TraitExpression(traitName: String) extends Expression with LeafNode {
- val dataType = typSystem.getDataType(classOf[TraitType], traitName)
-
- override def toString = traitName
- }
-
- def _trait(name: String) = new TraitExpression(name)
-
- case class IdExpression(name: String) extends Expression with LeafNode {
- override def toString = name
-
- override lazy val resolved = false
-
- override def dataType = throw new UnresolvedException(this, "id")
- }
-
- def id(name: String) = new IdExpression(name)
-
- case class UnresolvedFieldExpression(child: Expression, fieldName: String) extends Expression
- with UnaryNode {
- override def toString = s"${child}.$fieldName"
-
- override lazy val resolved = false
-
- override def dataType = throw new UnresolvedException(this, "field")
- }
-
- case class FieldExpression(fieldName: String, fieldInfo: FieldInfo, child: Option[Expression])
- extends Expression {
-
- def elemType(t: IDataType[_]): IDataType[_] = {
- if (t.getTypeCategory == TypeCategory.ARRAY) {
- val aT = t.asInstanceOf[ArrayType]
- if (aT.getElemType.getTypeCategory == TypeCategory.CLASS ||
- aT.getElemType.getTypeCategory == TypeCategory.STRUCT) {
- return aT.getElemType
- }
- }
- t
- }
-
- val children = if (child.isDefined) List(child.get) else Nil
- import scala.language.existentials
- lazy val dataType = {
- val t = {
- if (fieldInfo.traitName != null ) {
- typSystem.getDataType(classOf[TraitType], fieldInfo.traitName)
- } else if (!fieldInfo.isReverse) {
- fieldInfo.attrInfo.dataType()
- } else {
- fieldInfo.reverseDataType
- }
- }
- elemType(t)
- }
- override lazy val resolved: Boolean = true
-
- override def namedExpressions = if (child.isDefined) child.get.namedExpressions else Map()
-
- override def toString = {
- if (child.isDefined) {
- val sep = if (dataType.isInstanceOf[ClassType]) " " else "."
- s"${child.get}${sep}$fieldName"
- } else {
- fieldName
- }
- }
- }
-
- case class AliasExpression(child: Expression, alias: String) extends Expression with UnaryNode {
- override def namedExpressions = child.namedExpressions + (alias -> child)
-
- override def toString = s"$child as $alias"
-
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved child")
- }
- child.dataType
- }
- }
-
- case class BackReference(alias: String, reference: Expression, child: Option[Expression]) extends Expression {
- val children = if (child.isDefined) List(child.get) else Nil
- val dataType = reference.dataType
-
- override def namedExpressions = if (child.isDefined) child.get.namedExpressions else Map()
-
- override def toString = if (child.isDefined) s"${child.get} $alias" else alias
- }
-
- case class Literal[T](dataType: PrimitiveType[T], rawValue: Any) extends Expression with LeafNode {
- val value = if (rawValue == null) dataType.nullValue() else dataType.convert(rawValue, Multiplicity.REQUIRED)
-
- override def toString = value match {
- case s: String => s""""$s""""
- case x => x.toString
- }
- }
-
- def literal[T](typ: PrimitiveType[T], rawValue: Any) = new Literal[T](typ, rawValue)
-
- def boolean(rawValue: Any) = literal(DataTypes.BOOLEAN_TYPE, rawValue)
-
- def byte(rawValue: Any) = literal(DataTypes.BYTE_TYPE, rawValue)
-
- def short(rawValue: Any) = literal(DataTypes.SHORT_TYPE, rawValue)
-
- def int(rawValue: Any) = literal(DataTypes.INT_TYPE, rawValue)
-
- def long(rawValue: Any) = literal(DataTypes.LONG_TYPE, rawValue)
-
- def float(rawValue: Any) = literal(DataTypes.FLOAT_TYPE, rawValue)
-
- def double(rawValue: Any) = literal(DataTypes.DOUBLE_TYPE, rawValue)
-
- def bigint(rawValue: Any) = literal(DataTypes.BIGINTEGER_TYPE, rawValue)
-
- def bigdecimal(rawValue: Any) = literal(DataTypes.BIGDECIMAL_TYPE, rawValue)
-
- def string(rawValue: Any) = literal(DataTypes.STRING_TYPE, rawValue)
-
- case class ArithmeticExpression(symbol: String,
- left: Expression,
- right: Expression)
- extends BinaryExpression {
-
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved children")
- }
- TypeUtils.combinedType(left.dataType, right.dataType)
- }
- }
-
- case class isTraitLeafExpression(traitName: String, classExpression: Option[Expression] = None)
- extends Expression with LeafNode {
- // validate TraitName
- try {
- typSystem.getDataType(classOf[TraitType], traitName)
- } catch {
- case me: MetadataException => throw new ExpressionException(this, "not a TraitType", me)
- }
-
- override lazy val resolved = classExpression.isDefined
- lazy val dataType = {
-
- if (!resolved) {
- throw new UnresolvedException(this,
- s"cannot resolve isTrait application")
- }
-
- if (!classExpression.get.dataType.isInstanceOf[ClassType]) {
- throw new ExpressionException(this,
- s"Cannot apply isTrait on ${classExpression.get.dataType.getName}, it is not a ClassType")
- }
- DataTypes.BOOLEAN_TYPE
- }
-
- override def toString = s"${classExpression.getOrElse("")} is $traitName"
- }
-
- def isTrait(name: String) = new isTraitLeafExpression(name)
-
- case class isTraitUnaryExpression(traitName: String, child: Expression)
- extends Expression with UnaryNode {
- // validate TraitName
- typSystem.getDataType(classOf[TraitType], traitName)
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved child")
- }
- if (!child.dataType.isInstanceOf[ClassType]) {
- throw new ExpressionException(this,
- s"Cannot apply isTrait on ${child.dataType.getName}, it is not a ClassType")
- }
- DataTypes.BOOLEAN_TYPE
- }
-
- override def toString = s"$child is $traitName"
- }
-
- case class hasFieldLeafExpression(fieldName: String, classExpression: Option[Expression] = None)
- extends Expression with LeafNode {
-
- override lazy val resolved = classExpression.isDefined
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"Cannot apply hasField on ${classExpression.get.dataType.getName}, it is not a ClassType")
- }
- if (classExpression.isDefined && !TypeUtils.fieldMapping(classExpression.get.dataType).isDefined) {
- throw new ExpressionException(this, s"Cannot apply hasField on ${classExpression.get.dataType.getName}")
- }
- DataTypes.BOOLEAN_TYPE
- }
-
- override def toString = s"${classExpression.getOrElse("")} has $fieldName"
- }
-
- def hasField(name: String) = new hasFieldLeafExpression(name)
-
- case class hasFieldUnaryExpression(fieldName: String, child: Expression)
- extends Expression with UnaryNode {
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved child")
- }
- if (!TypeUtils.fieldMapping(child.dataType).isDefined) {
- throw new MetadataException(s"Cannot apply hasField on ${child.dataType.getName}")
- }
- DataTypes.BOOLEAN_TYPE
- }
-
- override def toString = s"$child has $fieldName"
- }
-
- case class ComparisonExpression(symbol: String,
- left: Expression,
- right: Expression)
- extends BinaryExpression {
-
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved children")
- }
- if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) {
- TypeUtils.combinedType(left.dataType, right.dataType)
- }
- DataTypes.BOOLEAN_TYPE
- }
- }
-
- case class LogicalExpression(symbol: String, children: List[Expression])
- extends Expression {
- assert(children.size > 0)
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved children")
- }
- children.foreach { childExpr =>
- if (childExpr.dataType != DataTypes.BOOLEAN_TYPE) {
- throw new MetadataException(
- s"Cannot apply logical operator '$symbol' on input of type '${childExpr.dataType}")
- }
- }
- DataTypes.BOOLEAN_TYPE
- }
-
- override def toString = children.mkString("", s" $symbol ", "")
- }
-
- case class FilterExpression(val child: Expression, val condExpr: Expression) extends Expression {
- val children = List(child, condExpr)
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved children")
- }
- if (condExpr.dataType != DataTypes.BOOLEAN_TYPE) {
- throw new ExpressionException(this, s"Filter condition '$condExpr' is not a boolean expression")
- }
- child.dataType
- }
-
- override def namedExpressions = child.namedExpressions ++ condExpr.namedExpressions
-
- override def toString = s"$child where $condExpr"
- }
-
- val GEN_COL_ALIAS_PREFIX = "_col"
-
- case class SelectExpression(child: Expression, selectList: List[Expression]) extends Expression {
-
- val children = List(child) ::: selectList
- lazy val selectListWithAlias = selectList.zipWithIndex map {
- case (s: AliasExpression, _) => s
- case (x, i) => new AliasExpression(x, s"${GEN_COL_ALIAS_PREFIX}_$i")
- }
-
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved children")
- }
- TypeUtils.createStructType(selectListWithAlias)
- }
-
- override def namedExpressions = child.namedExpressions ++ (selectList.flatMap(_.namedExpressions))
-
- override def toString = s"""$child select ${selectListWithAlias.mkString("", ", ", "")}"""
- }
-
- case class LoopExpression(val input: Expression, val loopingExpression: Expression,
- val times: Option[Literal[Integer]]) extends Expression {
- val children = List(input, loopingExpression)
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved children")
- }
- if (input.dataType.getTypeCategory != TypeCategory.CLASS) {
- throw new ExpressionException(this, s"Loop Expression applied to type : '${input.dataType.getName}';" +
- " loop can only be applied to Class Expressions")
- }
- if (input.dataType != loopingExpression.dataType) {
- throw new ExpressionException(this,
- s"Invalid Loop Expression; input and loopExpression dataTypes don't match: " +
- s"(${input.dataType.getName},${loopingExpression.dataType.getName}})")
- }
- input.dataType
- }
-
- override def namedExpressions = input.namedExpressions
-
- override def toString = {
- if (times.isDefined) s"$input loop ($loopingExpression) times ${times.get.value}"
- else s"$input loop ($loopingExpression)"
- }
- }
-
- case class TraitInstanceExpression(child: Expression)
- extends Expression with UnaryNode {
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved child")
- }
- if (!child.dataType.isInstanceOf[TraitType]) {
- throw new ExpressionException(this,
- s"Cannot apply instance on ${child.dataType.getName}, it is not a TraitType")
- }
- typSystem.getIdType.getStructType
- }
-
- override def toString = s"$child traitInstance"
- }
-
- case class InstanceExpression(child: Expression)
- extends Expression with UnaryNode {
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved child")
- }
- typSystem.getIdType.getStructType
- }
-
- override def toString = s"$child instance"
- }
-
- case class PathExpression(child: Expression)
- extends Expression with UnaryNode {
- lazy val dataType = {
- if (!resolved) {
- throw new UnresolvedException(this,
- s"datatype. Can not resolve due to unresolved child")
- }
- TypeUtils.ResultWithPathStruct.createType(this, child.dataType)
- }
-
- override def toString = s"$child withPath"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala
deleted file mode 100755
index 5684365..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import com.thinkaurelius.titan.core.TitanVertex
-import com.tinkerpop.blueprints.Direction
-import org.apache.hadoop.metadata.query.Expressions.{ComparisonExpression, ExpressionException}
-import org.apache.hadoop.metadata.query.TypeUtils.FieldInfo
-import org.apache.hadoop.metadata.typesystem.persistence.Id
-import org.apache.hadoop.metadata.typesystem.types.DataTypes._
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.typesystem.{ITypedInstance, ITypedReferenceableInstance}
-
-import scala.collection.JavaConversions._
-
-/**
- * Represents the Bridge between the QueryProcessor and the Graph Persistence scheme used.
- * Some of the behaviors captured are:
- * - how is type and id information stored in the Vertex that represents an [[ITypedReferenceableInstance]]
- * - how are edges representing trait and attribute relationships labelled.
- * - how are attribute names mapped to Property Keys in Vertices.
- *
- * This is a work in progress.
- */
-trait GraphPersistenceStrategies {
- /**
- * Name of attribute used to store typeName in vertex
- */
- def typeAttributeName: String
-
- /**
- * Name of attribute used to store super type names in vertex.
- */
- def superTypeAttributeName: String
-
- /**
- * Name of attribute used to store guid in vertex
- */
- def idAttributeName : String
-
- /**
- * Given a dataType and a reference attribute, how is edge labeled
- */
- def edgeLabel(iDataType: IDataType[_], aInfo: AttributeInfo): String
-
- def traitLabel(cls: IDataType[_], traitName: String): String
-
- def instanceToTraitEdgeDirection : String = "out"
- def traitToInstanceEdgeDirection = instanceToTraitEdgeDirection match {
- case "out" => "in"
- case "in" => "out"
- case x => x
- }
-
- /**
- * The propertyKey used to store the attribute in a Graph Vertex.
- * @param dataType
- * @param aInfo
- * @return
- */
- def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo): String
-
- /**
- * from a vertex for an [[ITypedReferenceableInstance]] get the traits that it has.
- * @param v
- * @return
- */
- def traitNames(v: TitanVertex): java.util.List[String]
-
- def edgeLabel(fInfo: FieldInfo): String = fInfo match {
- case FieldInfo(dataType, aInfo, null, null) => edgeLabel(dataType, aInfo)
- case FieldInfo(dataType, aInfo, reverseDataType, null) => edgeLabel(reverseDataType, aInfo)
- case FieldInfo(dataType, null, null, traitName) => traitLabel(dataType, traitName)
- }
-
- def fieldPrefixInSelect: String
-
- /**
- * extract the Id from a Vertex.
- * @param dataTypeNm the dataType of the instance that the given vertex represents
- * @param v
- * @return
- */
- def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id
-
- def constructInstance[U](dataType: IDataType[U], v: java.lang.Object): U
-
- def gremlinCompOp(op: ComparisonExpression) = op.symbol match {
- case "=" => "T.eq"
- case "!=" => "T.neq"
- case ">" => "T.gt"
- case ">=" => "T.gte"
- case "<" => "T.lt"
- case "<=" => "T.lte"
- case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin")
- }
-
- def loopObjectExpression(dataType: IDataType[_]) = {
- _typeTestExpression(dataType.getName, "it.object")
- }
-
- def addGraphVertexPrefix(preStatements : Traversable[String]) = !collectTypeInstancesIntoVar
-
- /**
- * Controls behavior of how instances of a Type are discovered.
- * - query is generated in a way that indexes are exercised using a local set variable across multiple lookups
- * - query is generated using an 'or' expression.
- *
- * '''This is a very bad idea: controlling query execution behavior via query generation.''' But our current
- * knowledge of seems to indicate we have no choice. See
- * [[https://groups.google.com/forum/#!topic/gremlin-users/n1oV86yr4yU discussion in Gremlin group]].
- * Also this seems a fragile solution, dependend on the memory requirements of the Set variable.
- * For now enabling via the '''collectTypeInstancesIntoVar''' behavior setting. Reverting back would require
- * setting this to false.
- *
- * Long term have to get to the bottom of Gremlin:
- * - there doesn't seem to be way to see the physical query plan. Maybe we should directly interface with Titan.
- * - At least from querying perspective a columnar db maybe a better route. Daniel Abadi did some good work
- * on showing how to use a columnar store as a Graph Db.
- *
- *
- * @return
- */
- def collectTypeInstancesIntoVar = true
-
- def typeTestExpression(typeName : String, intSeq : IntSequence) : Seq[String] = {
- if (collectTypeInstancesIntoVar)
- typeTestExpressionMultiStep(typeName, intSeq)
- else
- typeTestExpressionUsingFilter(typeName)
- }
-
- private def typeTestExpressionUsingFilter(typeName : String) : Seq[String] = {
- Seq(s"""filter${_typeTestExpression(typeName, "it")}""")
- }
-
- private def _typeTestExpression(typeName: String, itRef: String): String = {
- s"""{(${itRef}.'${typeAttributeName}' == '${typeName}') |
- |(${itRef}.'${superTypeAttributeName}' ?
- |${itRef}.'${superTypeAttributeName}'.contains('${typeName}') : false)}""".
- stripMargin.replace(System.getProperty("line.separator"), "")
- }
-
- private def typeTestExpressionMultiStep(typeName : String, intSeq : IntSequence) : Seq[String] = {
-
- val varName = s"_var_${intSeq.next}"
- Seq(
- newSetVar(varName),
- fillVarWithTypeInstances(typeName, varName),
- fillVarWithSubTypeInstances(typeName, varName),
- s"$varName._()"
- )
- }
-
- private def newSetVar(varName : String) = s"$varName = [] as Set"
-
- private def fillVarWithTypeInstances(typeName : String, fillVar : String) = {
- s"""g.V().has("${typeAttributeName}", "${typeName}").fill($fillVar)"""
- }
-
- private def fillVarWithSubTypeInstances(typeName : String, fillVar : String) = {
- s"""g.V().has("${superTypeAttributeName}", "${typeName}").fill($fillVar)"""
- }
-}
-
-object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
- val typeAttributeName = "typeName"
- val superTypeAttributeName = "superTypeNames"
- val idAttributeName = "guid"
-
- def edgeLabel(dataType: IDataType[_], aInfo: AttributeInfo) = s"${dataType.getName}.${aInfo.name}"
-
- val fieldPrefixInSelect = "it"
-
- def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName"
-
- def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = aInfo.name
-
- def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id =
- new Id(v.getId.toString, 0, dataTypeNm)
-
- def traitNames(v: TitanVertex): java.util.List[String] = {
- val s = v.getProperty[String]("traitNames")
- if (s != null) {
- Seq[String](s.split(","): _*)
- } else {
- Seq()
- }
- }
-
- def constructInstance[U](dataType: IDataType[U], v: AnyRef): U = {
- dataType.getTypeCategory match {
- case DataTypes.TypeCategory.PRIMITIVE => dataType.convert(v, Multiplicity.OPTIONAL)
- case DataTypes.TypeCategory.STRUCT
- if dataType.getName == TypeSystem.getInstance().getIdType.getName => {
- val sType = dataType.asInstanceOf[StructType]
- val sInstance = sType.createInstance()
- val tV = v.asInstanceOf[TitanVertex]
- sInstance.set(TypeSystem.getInstance().getIdType.typeNameAttrName,
- tV.getProperty[java.lang.String](typeAttributeName))
- sInstance.set(TypeSystem.getInstance().getIdType.idAttrName,
- tV.getProperty[java.lang.String](idAttributeName))
- dataType.convert(sInstance, Multiplicity.OPTIONAL)
- }
- case DataTypes.TypeCategory.STRUCT => {
- val sType = dataType.asInstanceOf[StructType]
- val sInstance = sType.createInstance()
- loadStructInstance(sType, sInstance, v.asInstanceOf[TitanVertex])
- dataType.convert(sInstance, Multiplicity.OPTIONAL)
- }
- case DataTypes.TypeCategory.TRAIT => {
- val tType = dataType.asInstanceOf[TraitType]
- val tInstance = tType.createInstance()
- /*
- * this is not right, we should load the Instance associated with this trait.
- * for now just loading the trait struct.
- */
- loadStructInstance(tType, tInstance, v.asInstanceOf[TitanVertex])
- dataType.convert(tInstance, Multiplicity.OPTIONAL)
- }
- case DataTypes.TypeCategory.CLASS => {
- val cType = dataType.asInstanceOf[ClassType]
- val cInstance = constructClassInstance(dataType.asInstanceOf[ClassType], v.asInstanceOf[TitanVertex])
- dataType.convert(cInstance, Multiplicity.OPTIONAL)
- }
- case DataTypes.TypeCategory.ENUM => dataType.convert(v, Multiplicity.OPTIONAL)
- case x => throw new UnsupportedOperationException(s"load for ${dataType} not supported")
- }
- }
-
- def loadStructInstance(dataType: IConstructableType[_, _ <: ITypedInstance],
- typInstance: ITypedInstance, v: TitanVertex): Unit = {
- import scala.collection.JavaConversions._
- dataType.fieldMapping().fields.foreach { t =>
- val fName = t._1
- val aInfo = t._2
- loadAttribute(dataType, aInfo, typInstance, v)
- }
- }
-
- def constructClassInstance(dataType: ClassType, v: TitanVertex): ITypedReferenceableInstance = {
- val id = getIdFromVertex(dataType.name, v)
- val tNms = traitNames(v)
- val cInstance = dataType.createInstance(id, tNms: _*)
- // load traits
- tNms.foreach { tNm =>
- val tLabel = traitLabel(dataType, tNm)
- val edges = v.getEdges(Direction.OUT, tLabel)
- val tVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex]
- val tType = TypeSystem.getInstance().getDataType[TraitType](classOf[TraitType], tNm)
- val tInstance = cInstance.getTrait(tNm).asInstanceOf[ITypedInstance]
- loadStructInstance(tType, tInstance, tVertex)
- }
- loadStructInstance(dataType, cInstance, v)
- cInstance
- }
-
- def loadAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex): Unit = {
- aInfo.dataType.getTypeCategory match {
- case DataTypes.TypeCategory.PRIMITIVE => loadPrimitiveAttribute(dataType, aInfo, i, v)
- case DataTypes.TypeCategory.ENUM => loadEnumAttribute(dataType, aInfo, i, v)
- case DataTypes.TypeCategory.ARRAY =>
- throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
- case DataTypes.TypeCategory.MAP =>
- throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
- case DataTypes.TypeCategory.STRUCT => loadStructAttribute(dataType, aInfo, i, v)
- case DataTypes.TypeCategory.TRAIT =>
- throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
- case DataTypes.TypeCategory.CLASS => loadStructAttribute(dataType, aInfo, i, v)
- }
- }
-
- private def loadEnumAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex)
- : Unit = {
- val fName = fieldNameInVertex(dataType, aInfo)
- i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName))
- }
-
- private def loadPrimitiveAttribute(dataType: IDataType[_], aInfo: AttributeInfo,
- i: ITypedInstance, v: TitanVertex): Unit = {
- val fName = fieldNameInVertex(dataType, aInfo)
- aInfo.dataType() match {
- case x: BooleanType => i.setBoolean(aInfo.name, v.getProperty[java.lang.Boolean](fName))
- case x: ByteType => i.setByte(aInfo.name, v.getProperty[java.lang.Byte](fName))
- case x: ShortType => i.setShort(aInfo.name, v.getProperty[java.lang.Short](fName))
- case x: IntType => i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName))
- case x: LongType => i.setLong(aInfo.name, v.getProperty[java.lang.Long](fName))
- case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName))
- case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName))
- case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName))
- case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
- }
- }
-
- private def loadStructAttribute(dataType: IDataType[_], aInfo: AttributeInfo,
- i: ITypedInstance, v: TitanVertex): Unit = {
- val eLabel = edgeLabel(FieldInfo(dataType, aInfo, null))
- val edges = v.getEdges(Direction.OUT, eLabel)
- val sVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex]
- if (aInfo.dataType().getTypeCategory == DataTypes.TypeCategory.STRUCT) {
- val sType = aInfo.dataType().asInstanceOf[StructType]
- val sInstance = sType.createInstance()
- loadStructInstance(sType, sInstance, sVertex)
- i.set(aInfo.name, sInstance)
- } else {
- val cInstance = constructClassInstance(aInfo.dataType().asInstanceOf[ClassType], sVertex)
- i.set(aInfo.name, cInstance)
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala
deleted file mode 100755
index 6a77bc3..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
-
-import com.thinkaurelius.titan.core.TitanGraph
-import com.tinkerpop.pipes.util.structures.Row
-import org.apache.hadoop.metadata.query.TypeUtils.ResultWithPathStruct
-import org.apache.hadoop.metadata.typesystem.json._
-import org.apache.hadoop.metadata.typesystem.types._
-import org.json4s._
-import org.json4s.native.Serialization._
-
-import scala.language.existentials
-
-case class GremlinQueryResult(query: String,
- resultDataType: IDataType[_],
- rows: List[_]) {
- def toJson = JsonHelper.toJson(this)
-}
-
-class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: TitanGraph) {
-
- val manager: ScriptEngineManager = new ScriptEngineManager
- val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
- val bindings: Bindings = engine.createBindings
- bindings.put("g", g)
-
- /**
- *
- * @param gResultObj is the object returned from gremlin. This must be a List
- * @param qryResultObj is the object constructed for the output w/o the Path.
- * @return a ResultWithPathStruct
- */
- def addPathStruct(gResultObj : AnyRef, qryResultObj : Any) : Any = {
- if ( !qry.isPathExpresion) {
- qryResultObj
- } else {
- import scala.collection.JavaConversions._
- import scala.collection.JavaConverters._
- val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init
-
- val oPaths = iPaths.map { p =>
- persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, p)
- }.toList.asJava
- val sType = qry.expr.dataType.asInstanceOf[StructType]
- val sInstance = sType.createInstance()
- sInstance.set(ResultWithPathStruct.pathAttrName, oPaths)
- sInstance.set(ResultWithPathStruct.resultAttrName, qryResultObj)
- sInstance
- }
- }
-
- def instanceObject(v : AnyRef) : AnyRef = {
- if ( qry.isPathExpresion ) {
- import scala.collection.JavaConversions._
- v.asInstanceOf[java.util.List[AnyRef]].last
- } else {
- v
- }
- }
-
- def evaluate(): GremlinQueryResult = {
- import scala.collection.JavaConversions._
- val rType = qry.expr.dataType
- val oType = if (qry.isPathExpresion) qry.expr.children(0).dataType else rType
- val rawRes = engine.eval(qry.queryStr, bindings)
-
- if (!qry.hasSelectList) {
- val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { v =>
- val iV = instanceObject(v)
- val o = persistenceStrategy.constructInstance(oType, iV)
- addPathStruct(v, o)
- }
- GremlinQueryResult(qry.expr.toString, rType, rows.toList)
- } else {
- val sType = oType.asInstanceOf[StructType]
- val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { r =>
- val rV = instanceObject(r).asInstanceOf[Row[java.util.List[AnyRef]]]
- val sInstance = sType.createInstance()
- val selExpr =
- (if (qry.isPathExpresion) qry.expr.children(0) else qry.expr).
- asInstanceOf[Expressions.SelectExpression]
- selExpr.selectListWithAlias.foreach { aE =>
- val cName = aE.alias
- val (src, idx) = qry.resultMaping(cName)
- val v = rV.getColumn(src).get(idx)
- sInstance.set(cName, persistenceStrategy.constructInstance(aE.dataType, v))
- }
- addPathStruct(r, sInstance)
- }
- GremlinQueryResult(qry.expr.toString, rType, rows.toList)
- }
-
- }
-}
-
-object JsonHelper {
-
- class GremlinQueryResultSerializer()
- extends Serializer[GremlinQueryResult] {
- def deserialize(implicit format: Formats) = {
- throw new UnsupportedOperationException("Deserialization of GremlinQueryResult not supported")
- }
-
- def serialize(implicit f: Formats) = {
- case GremlinQueryResult(query, rT, rows) =>
- JObject(JField("query", JString(query)),
- JField("dataType", TypesSerialization.toJsonValue(rT)(f)),
- JField("rows", Extraction.decompose(rows)(f))
- )
- }
- }
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer +
- new GremlinQueryResultSerializer
-
- def toJson(r: GremlinQueryResult): String = {
- writePretty(r)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala
deleted file mode 100755
index 1395f6d..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import org.apache.hadoop.metadata.query.Expressions._
-import org.apache.hadoop.metadata.typesystem.types.DataTypes.TypeCategory
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
-trait IntSequence {
- def next: Int
-}
-
-case class GremlinQuery(expr: Expression, queryStr: String, resultMaping: Map[String, (String, Int)]) {
-
- def hasSelectList = resultMaping != null
-
- def isPathExpresion = expr.isInstanceOf[PathExpression]
-}
-
-trait SelectExpressionHandling {
-
- /**
- * To aide in gremlinQuery generation add an alias to the input of SelectExpressions
- */
- class AddAliasToSelectInput extends PartialFunction[Expression, Expression] {
-
- private var idx = 0
-
- def isDefinedAt(e: Expression) = true
-
- class DecorateFieldWithAlias(aliasE: AliasExpression)
- extends PartialFunction[Expression, Expression] {
- def isDefinedAt(e: Expression) = true
-
- def apply(e: Expression) = e match {
- case fe@FieldExpression(fieldName, fInfo, None) =>
- FieldExpression(fieldName, fInfo, Some(BackReference(aliasE.alias, aliasE.child, None)))
- case _ => e
- }
- }
-
- def apply(e: Expression) = e match {
- case SelectExpression(aliasE@AliasExpression(_, _), selList) => {
- idx = idx + 1
- SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE))))
- }
- case SelectExpression(child, selList) => {
- idx = idx + 1
- val aliasE = AliasExpression(child, s"_src$idx")
- SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE))))
- }
- case _ => e
- }
- }
-
- def getSelectExpressionSrc(e: Expression): List[String] = {
- val l = ArrayBuffer[String]()
- e.traverseUp {
- case BackReference(alias, _, _) => l += alias
- }
- l.toSet.toList
- }
-
- def validateSelectExprHaveOneSrc: PartialFunction[Expression, Unit] = {
- case SelectExpression(_, selList) => {
- selList.foreach { se =>
- val srcs = getSelectExpressionSrc(se)
- if (srcs.size > 1) {
- throw new GremlinTranslationException(se, "Only one src allowed in a Select Expression")
- }
- }
- }
- }
-
- def groupSelectExpressionsBySrc(sel: SelectExpression): mutable.LinkedHashMap[String, List[Expression]] = {
- val m = mutable.LinkedHashMap[String, List[Expression]]()
- sel.selectListWithAlias.foreach { se =>
- val l = getSelectExpressionSrc(se.child)
- if (!m.contains(l(0))) {
- m(l(0)) = List()
- }
- m(l(0)) = m(l(0)) :+ se.child
- }
- m
- }
-
- /**
- * For each Output Column in the SelectExpression compute the ArrayList(Src) this maps to and the position within
- * this list.
- * @param sel
- * @return
- */
- def buildResultMapping(sel: SelectExpression): Map[String, (String, Int)] = {
- val srcToExprs = groupSelectExpressionsBySrc(sel)
- val m = new mutable.HashMap[String, (String, Int)]
- sel.selectListWithAlias.foreach { se =>
- val src = getSelectExpressionSrc(se.child)(0)
- val srcExprs = srcToExprs(src)
- var idx = srcExprs.indexOf(se.child)
- m(se.alias) = (src, idx)
- }
- m.toMap
- }
-
-}
-
-class GremlinTranslationException(expr: Expression, reason: String) extends
-ExpressionException(expr, s"Unsupported Gremlin translation: $reason")
-
-class GremlinTranslator(expr: Expression,
- gPersistenceBehavior: GraphPersistenceStrategies)
- extends SelectExpressionHandling {
-
- val preStatements = ArrayBuffer[String]()
- val postStatements = ArrayBuffer[String]()
-
- val wrapAndRule: PartialFunction[Expression, Expression] = {
- case f: FilterExpression if !f.condExpr.isInstanceOf[LogicalExpression] =>
- FilterExpression(f.child, new LogicalExpression("and", List(f.condExpr)))
- }
-
- val validateComparisonForm: PartialFunction[Expression, Unit] = {
- case c@ComparisonExpression(_, left, right) =>
- if (!left.isInstanceOf[FieldExpression]) {
- throw new GremlinTranslationException(c, s"lhs of comparison is not a field")
- }
- if (!right.isInstanceOf[Literal[_]]) {
- throw new GremlinTranslationException(c,
- s"rhs of comparison is not a literal")
- }
- ()
- }
-
- val counter = new IntSequence {
- var i: Int = -1;
-
- def next: Int = {
- i += 1; i
- }
- }
-
- def addAliasToLoopInput(c: IntSequence = counter): PartialFunction[Expression, Expression] = {
- case l@LoopExpression(aliasE@AliasExpression(_, _), _, _) => l
- case l@LoopExpression(inputExpr, loopExpr, t) => {
- val aliasE = AliasExpression(inputExpr, s"_loop${c.next}")
- LoopExpression(aliasE, loopExpr, t)
- }
- }
-
- def instanceClauseToTop(topE : Expression) : PartialFunction[Expression, Expression] = {
- case le : LogicalExpression if (le fastEquals topE) => {
- le.instance()
- }
- case ce : ComparisonExpression if (ce fastEquals topE) => {
- ce.instance()
- }
- case he : hasFieldUnaryExpression if (he fastEquals topE) => {
- he.instance()
- }
- }
-
- def traitClauseWithInstanceForTop(topE : Expression) : PartialFunction[Expression, Expression] = {
- case te : TraitExpression if (te fastEquals topE) => {
- val theTrait = te.as("theTrait")
- val theInstance = theTrait.traitInstance().as("theInstance")
- val outE =
- theInstance.select(id("theTrait").as("traitDetails"),
- id("theInstance").as("instanceInfo"))
- QueryProcessor.validate(outE)
- }
- }
-
- def typeTestExpression(typeName : String) : String = {
- val stats = gPersistenceBehavior.typeTestExpression(typeName, counter)
- preStatements ++= stats.init
- stats.last
- }
-
- private def genQuery(expr: Expression, inSelect: Boolean): String = expr match {
- case ClassExpression(clsName) =>
- typeTestExpression(clsName)
- case TraitExpression(clsName) =>
- typeTestExpression(clsName)
- case fe@FieldExpression(fieldName, fInfo, child) if fe.dataType.getTypeCategory == TypeCategory.PRIMITIVE => {
- val fN = "\"" + gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo) + "\""
- child match {
- case Some(e) => s"${genQuery(e, inSelect)}.$fN"
- case None => s"$fN"
- }
- }
- case fe@FieldExpression(fieldName, fInfo, child)
- if fe.dataType.getTypeCategory == TypeCategory.CLASS || fe.dataType.getTypeCategory == TypeCategory.STRUCT => {
- val direction = if (fInfo.isReverse) "in" else "out"
- val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo)
- val step = s"""$direction("$edgeLbl")"""
- child match {
- case Some(e) => s"${genQuery(e, inSelect)}.$step"
- case None => step
- }
- }
- case fe@FieldExpression(fieldName, fInfo, child)
- if fInfo.traitName != null => {
- val direction = gPersistenceBehavior.instanceToTraitEdgeDirection
- val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo)
- val step = s"""$direction("$edgeLbl")"""
- child match {
- case Some(e) => s"${genQuery(e, inSelect)}.$step"
- case None => step
- }
- }
- case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => {
- val fieldGremlinExpr = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}"
- ch match {
- case Some(child) => {
- s"""${genQuery(child, inSelect)}.has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
- }
- case None => s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
- }
- }
- case fil@FilterExpression(child, condExpr) => {
- s"${genQuery(child, inSelect)}.${genQuery(condExpr, inSelect)}"
- }
- case l@LogicalExpression(symb, children) => {
- s"""$symb${children.map("_()." + genQuery(_, inSelect)).mkString("(", ",", ")")}"""
- }
- case sel@SelectExpression(child, selList) => {
- val m = groupSelectExpressionsBySrc(sel)
- var srcNamesList: List[String] = List()
- var srcExprsList: List[List[String]] = List()
- val it = m.iterator
- while (it.hasNext) {
- val (src, selExprs) = it.next
- srcNamesList = srcNamesList :+ s""""$src""""
- srcExprsList = srcExprsList :+ selExprs.map { selExpr =>
- genQuery(selExpr, true)
- }
- }
- val srcNamesString = srcNamesList.mkString("[", ",", "]")
- val srcExprsStringList = srcExprsList.map {
- _.mkString("[", ",", "]")
- }
- val srcExprsString = srcExprsStringList.foldLeft("")(_ + "{" + _ + "}")
- s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString"
- }
- case loop@LoopExpression(input, loopExpr, t) => {
- val inputQry = genQuery(input, inSelect)
- val loopingPathGExpr = genQuery(loopExpr, inSelect)
- val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")"""
- val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{true}"
- val loopObjectGExpr = gPersistenceBehavior.loopObjectExpression(input.dataType)
- s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}"""
- }
- case BackReference(alias, _, _) =>
- if (inSelect) gPersistenceBehavior.fieldPrefixInSelect else s"""back("$alias")"""
- case AliasExpression(child, alias) => s"""${genQuery(child, inSelect)}.as("$alias")"""
- case isTraitLeafExpression(traitName, Some(clsExp)) =>
- s"""out("${gPersistenceBehavior.traitLabel(clsExp.dataType, traitName)}")"""
- case isTraitUnaryExpression(traitName, child) =>
- s"""out("${gPersistenceBehavior.traitLabel(child.dataType, traitName)}")"""
- case hasFieldLeafExpression(fieldName, Some(clsExp)) =>
- s"""has("$fieldName")"""
- case hasFieldUnaryExpression(fieldName, child) =>
- s"""${genQuery(child, inSelect)}.has("$fieldName")"""
- case ArithmeticExpression(symb, left, right) => s"${genQuery(left, inSelect)} $symb ${genQuery(right, inSelect)}"
- case l: Literal[_] => l.toString
- case in@TraitInstanceExpression(child) => {
- val direction = gPersistenceBehavior.traitToInstanceEdgeDirection
- s"${genQuery(child, inSelect)}.$direction()"
- }
- case in@InstanceExpression(child) => {
- s"${genQuery(child, inSelect)}"
- }
- case pe@PathExpression(child) => {
- s"${genQuery(child, inSelect)}.path"
- }
- case x => throw new GremlinTranslationException(x, "expression not yet supported")
- }
-
- def genFullQuery(expr: Expression): String = {
- var q = genQuery(expr, false)
-
- if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) {
- q = s"g.V.$q"
- }
-
- q = s"$q.toList()"
-
- q = (preStatements ++ Seq(q) ++ postStatements).mkString("", ";", "")
- /*
- * the L:{} represents a groovy code block; the label is needed
- * to distinguish it from a groovy closure.
- */
- s"L:{$q}"
- }
-
- def translate(): GremlinQuery = {
- var e1 = expr.transformUp(wrapAndRule)
-
- e1.traverseUp(validateComparisonForm)
-
- e1 = e1.transformUp(new AddAliasToSelectInput)
- e1.traverseUp(validateSelectExprHaveOneSrc)
- e1 = e1.transformUp(addAliasToLoopInput())
- e1 = e1.transformUp(instanceClauseToTop(e1))
- e1 = e1.transformUp(traitClauseWithInstanceForTop(e1))
-
- e1 match {
- case e1: SelectExpression => {
- val rMap = buildResultMapping(e1)
- GremlinQuery(e1, genFullQuery(e1), rMap)
- }
- case pe@PathExpression(se@SelectExpression(child, selectList)) => {
- val rMap = buildResultMapping(se)
- GremlinQuery(e1, genFullQuery(e1), rMap)
- }
- case e1 => GremlinQuery(e1, genFullQuery(e1), null)
- }
-
- }
-
- /*
- * Translation Issues:
- * 1. back references in filters. For e.g. testBackreference: 'DB as db Table where (db.name = "Reporting")'
- * this is translated to:
- * g.V.has("typeName","DB").as("db").in("Table.db").and(_().back("db").has("name", T.eq, "Reporting")).map().toList()
- * But the '_().back("db") within the and is ignored, the has condition is applied on the current element.
- * The solution is to to do predicate pushdown and apply the filter immediately on top of the referred Expression.
- */
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala
deleted file mode 100755
index 66628c0..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import org.apache.hadoop.metadata.query.Expressions._
-
-import scala.util.parsing.combinator.lexical.StdLexical
-import scala.util.parsing.combinator.syntactical.StandardTokenParsers
-import scala.util.parsing.combinator.{ImplicitConversions, PackratParsers}
-import scala.util.parsing.input.CharArrayReader._
-
-trait QueryKeywords {
- this: StandardTokenParsers =>
-
- import scala.language.implicitConversions
-
- protected case class Keyword(str: String)
-
- protected implicit def asParser(k: Keyword): Parser[String] = k.str
-
- protected val LPAREN = Keyword("(")
- protected val RPAREN = Keyword(")")
- protected val EQ = Keyword("=")
- protected val LT = Keyword("<")
- protected val GT = Keyword(">")
- protected val NEQ = Keyword("!=")
- protected val LTE = Keyword("<=")
- protected val GTE = Keyword(">=")
- protected val COMMA = Keyword(",")
- protected val AND = Keyword("and")
- protected val OR = Keyword("or")
- protected val PLUS = Keyword("+")
- protected val MINUS = Keyword("-")
- protected val STAR = Keyword("*")
- protected val DIV = Keyword("/")
- protected val DOT = Keyword(".")
-
- protected val SELECT = Keyword("select")
- protected val FROM = Keyword("from")
- protected val WHERE = Keyword("where")
- protected val GROUPBY = Keyword("groupby")
- protected val LOOP = Keyword("loop")
- protected val ISA = Keyword("isa")
- protected val IS = Keyword("is")
- protected val HAS = Keyword("has")
- protected val AS = Keyword("as")
- protected val TIMES = Keyword("times")
- protected val WITHPATH = Keyword("withPath")
-}
-
-trait ExpressionUtils {
-
- def loop(input: Expression, l: (Expression, Option[Literal[Integer]], Option[String])) = l match {
- case (c, None, None) => input.loop(c)
- case (c, t, None) => input.loop(c, t.get)
- case (c, None, Some(a)) => input.loop(c).as(a)
- case (c, t, Some(a)) => input.loop(c, t.get).as(a)
- }
-
- def select(input: Expression, s: List[(Expression, Option[String])]) = {
- val selList = s.map { t =>
- t._2 match {
- case None => t._1
- case _ => t._1.as(t._2.get)
- }
- }
- input.select(selList: _*)
- }
-
- def leftmostId(e: Expression) = {
- var le: IdExpression = null
- e.traverseUp { case i: IdExpression if le == null => le = i}
- le
- }
-
- def notIdExpression = new PartialFunction[Expression, Expression] {
- def isDefinedAt(x: Expression): Boolean = !x.isInstanceOf[IdExpression]
-
- def apply(e: Expression) = e
- }
-
- def replaceIdWithField(id: IdExpression, fe: UnresolvedFieldExpression): PartialFunction[Expression, Expression] = {
- case e: IdExpression if e == id => fe
- }
-
- def merge(snglQuery1: Expression, sngQuery2: Expression): Expression = {
- val leftSrcId = leftmostId(sngQuery2)
- sngQuery2.transformUp(replaceIdWithField(leftSrcId, snglQuery1.field(leftSrcId.name)))
- }
-}
-
-class QueryParser extends StandardTokenParsers with QueryKeywords with ExpressionUtils with PackratParsers {
-
- import scala.language.higherKinds
-
- private val reservedWordsDelims: Seq[String] = this.
- getClass.getMethods.filter(_.getReturnType == classOf[Keyword]).map(_.invoke(this).asInstanceOf[Keyword].str)
-
- private val (queryreservedWords: Seq[String], querydelims: Seq[String]) =
- reservedWordsDelims.partition(s => s.charAt(0).isLetter)
-
- override val lexical = new QueryLexer(queryreservedWords, querydelims)
-
- def apply(input: String): Either[NoSuccess, Expression] = {
- phrase(queryWithPath)(new lexical.Scanner(input)) match {
- case Success(r, x) => Right(r)
- case f@Failure(m, x) => Left(f)
- case e@Error(m, x) => Left(e)
- }
- }
-
- def queryWithPath = query ~ opt(WITHPATH) ^^ {
- case q ~ None => q
- case q ~ p => q.path()
- }
-
- def query: Parser[Expression] = rep1sep(singleQuery, opt(COMMA)) ^^ { l => l match {
- case h :: Nil => h
- case h :: t => t.foldLeft(h)(merge(_, _))
- }
- }
-
- def singleQuery = singleQrySrc ~ opt(loopExpression) ~ opt(selectClause) ^^ {
- case s ~ None ~ None => s
- case s ~ l ~ None => loop(s, l.get)
- case s ~ l ~ sel if l.isDefined => select(loop(s, l.get), sel.get)
- case s ~ None ~ sel => select(s, sel.get)
- }
-
- /**
- * A SingleQuerySrc can have the following forms:
- * 1. FROM id [WHERE] [expr] -> from optionally followed by a filter
- * 2. WHERE expr -> where clause, FROM is assumed to be the leftmost Id in the where clause
- * 3. expr (that is not an IdExpression) -> where clause, FROM is assumed to be the leftmost Id in the expr
- * 4. Id [WHERE] [expr] -> from optionally followed by a filter
- *
- * @return
- */
- def singleQrySrc: Parser[Expression] = FROM ~ fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ {
- case f ~ i ~ w ~ None => i
- case f ~ i ~ w ~ c => i.where(c.get)
- } |
- WHERE ~ (expr ^? notIdExpression) ^^ { case w ~ e => {
- val lId = leftmostId(e)
- if (lId == null) {
- failure("cannot infer Input from the where clause")
- }
- lId.where(e)
- }
- } |
- expr ^? notIdExpression ^^ { case e => {
- val lId = leftmostId(e)
- if (lId == null) {
- failure("cannot infer Input from the where clause")
- }
- lId.where(e)
- }
- } |
- fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ {
- case i ~ w ~ None => i
- case i ~ w ~ c => i.where(c.get)
- }
-
- def fromSrc = identifier ~ AS ~ alias ^^ { case s ~ a ~ al => s.as(al)} |
- identifier
-
-
- def loopExpression: Parser[(Expression, Option[Literal[Integer]], Option[String])] =
- LOOP ~ (LPAREN ~> query <~ RPAREN) ~ opt(intConstant <~ TIMES) ~ opt(AS ~> alias) ^^ {
- case l ~ e ~ None ~ a => (e, None, a)
- case l ~ e ~ Some(i) ~ a => (e, Some(int(i)), a)
- }
-
- def selectClause: Parser[List[(Expression, Option[String])]] = SELECT ~ rep1sep(selectExpression, COMMA) ^^ {
- case s ~ cs => cs
- }
-
- def selectExpression: Parser[(Expression, Option[String])] = expr ~ opt(AS ~> alias) ^^ {
- case e ~ a => (e, a)
- }
-
- def expr: Parser[Expression] = compE ~ opt(rep(exprRight)) ^^ {
- case l ~ None => l
- case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.logicalOp(r._1)(r._2)}
- }
-
- def exprRight = (AND | OR) ~ compE ^^ { case op ~ c => (op, c)}
-
- def compE =
- arithE ~ (LT | LTE | EQ | NEQ | GT | GTE) ~ arithE ^^ { case l ~ op ~ r => l.compareOp(op)(r)} |
- arithE ~ (ISA | IS) ~ ident ^^ { case l ~ i ~ t => l.isTrait(t)} |
- arithE ~ HAS ~ ident ^^ { case l ~ i ~ f => l.hasField(f)} |
- arithE
-
- def arithE = multiE ~ opt(rep(arithERight)) ^^ {
- case l ~ None => l
- case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)}
- }
-
- def arithERight = (PLUS | MINUS) ~ multiE ^^ { case op ~ r => (op, r)}
-
- def multiE = atomE ~ opt(rep(multiERight)) ^^ {
- case l ~ None => l
- case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)}
- }
-
- def multiERight = (STAR | DIV) ~ atomE ^^ { case op ~ r => (op, r)}
-
-
- def atomE = literal | identifier | LPAREN ~> expr <~ RPAREN
-
- def identifier = rep1sep(ident, DOT) ^^ { l => l match {
- case h :: Nil => id(h)
- case h :: t => {
- t.foldLeft(id(h).asInstanceOf[Expression])(_.field(_))
- }
- }
- }
-
- def alias = ident | stringLit
-
- def literal = booleanConstant ^^ {
- boolean(_)
- } |
- intConstant ^^ {
- int(_)
- } |
- longConstant ^^ {
- long(_)
- } |
- floatConstant ^^ {
- float(_)
- } |
- doubleConstant ^^ {
- double(_)
- } |
- stringLit ^^ {
- string(_)
- }
-
- def booleanConstant: Parser[String] =
- elem("int", _.isInstanceOf[lexical.BooleanLiteral]) ^^ (_.chars)
-
- def intConstant: Parser[String] =
- elem("int", _.isInstanceOf[lexical.IntLiteral]) ^^ (_.chars)
-
- def longConstant: Parser[String] =
- elem("int", _.isInstanceOf[lexical.LongLiteral]) ^^ (_.chars)
-
- def floatConstant: Parser[String] =
- elem("int", _.isInstanceOf[lexical.FloatLiteral]) ^^ (_.chars)
-
- def doubleConstant: Parser[String] =
- elem("int", _.isInstanceOf[lexical.DoubleLiteral]) ^^ (_.chars)
-
-}
-
-class QueryLexer(val keywords: Seq[String], val delims: Seq[String]) extends StdLexical with ImplicitConversions {
-
- case class BooleanLiteral(chars: String) extends Token {
- override def toString = chars
- }
-
- case class IntLiteral(chars: String) extends Token {
- override def toString = chars
- }
-
- case class LongLiteral(chars: String) extends Token {
- override def toString = chars
- }
-
- case class FloatLiteral(chars: String) extends Token {
- override def toString = chars
- }
-
- case class DoubleLiteral(chars: String) extends Token {
- override def toString = chars
- }
-
- reserved ++= keywords.flatMap(w => allCaseVersions(w))
-
- delimiters ++= delims
-
- override lazy val token: Parser[Token] =
- (
- (trueP | falseP)
- | longConstant ^^ LongLiteral
- | intConstant ^^ IntLiteral
- | floatConstant ^^ FloatLiteral
- | dubConstant ^^ DoubleLiteral
- | identifier ^^ processIdent
- | string ^^ StringLit
- | EofCh ^^^ EOF
- | '\'' ~> failure("unclosed string literal")
- | '"' ~> failure("unclosed string literal")
- | delim
- | '.' ^^^ new Keyword(".")
- | failure("illegal character")
- )
-
- override def identChar = letter | elem('_')
-
- def identifier = identChar ~ (identChar | digit).* ^^ { case first ~ rest => (first :: rest).mkString} |
- '`' ~> chrExcept('`', '\n', EofCh).* <~ '`' ^^ {
- _ mkString ""
- }
-
- override def whitespace: Parser[Any] =
- (whitespaceChar
- | '/' ~ '*' ~ comment
- | '/' ~ '/' ~ chrExcept(EofCh, '\n').*
- | '#' ~ chrExcept(EofCh, '\n').*
- | '/' ~ '*' ~ failure("unclosed comment")
- ).*
-
- protected override def comment: Parser[Any] = (
- commentChar.* ~ '*' ~ '/'
- )
-
- protected def commentChar = chrExcept(EofCh, '*') | '*' ~ not('/')
-
- def string = '\"' ~> chrExcept('\"', '\n', EofCh).* <~ '\"' ^^ {
- _ mkString ""
- } |
- '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^ {
- _ mkString ""
- }
-
- def zero: Parser[String] = '0' ^^^ "0"
-
- def nonzero = elem("nonzero digit", d => d.isDigit && d != '0')
-
- def sign = elem("sign character", d => d == '-' || d == '+')
-
- def exponent = elem("exponent character", d => d == 'e' || d == 'E')
-
-
- def intConstant = opt(sign) ~> zero | intList
-
- def intList = opt(sign) ~ nonzero ~ rep(digit) ^^ { case s ~ x ~ y => (optString("", s) :: x :: y) mkString ""}
-
- def fracPart: Parser[String] = '.' ~> rep(digit) ^^ { r =>
- "." + (r mkString "")
- }
-
- def expPart = exponent ~ opt(sign) ~ rep1(digit) ^^ { case e ~ s ~ d =>
- e.toString + optString("", s) + d.mkString("")
- }
-
- def dubConstant = opt(sign) ~ digit.+ ~ fracPart ~ opt(expPart) ^^ {
- case s ~ i ~ f ~ e => {
- optString("", s) + (i mkString "") + f + optString("", e)
- }
- }
-
- def floatConstant = opt(sign) ~ digit.* ~ fracPart ~ 'f' ^^ { case s ~ i ~ fr ~ f =>
- optString("", s) + i + fr
- } | opt(sign) ~ digit.+ ~ opt(fracPart) ~ 'f' ^^ { case s ~ i ~ fr ~ f =>
- optString("", s) + i + optString("", fr)
- }
-
- def longConstant = intConstant ~ 'l' ^^ { case i ~ l => i}
-
- def trueP = 't' ~ 'r' ~ 'u' ~ 'e' ^^^ BooleanLiteral("true")
-
- def falseP = 'f' ~ 'a' ~ 'l' ~ 's' ~ 'e' ^^^ BooleanLiteral("false")
-
- private def optString[A](pre: String, a: Option[A]) = a match {
- case Some(x) => pre + x.toString
- case None => ""
- }
-
- /** Generate all variations of upper and lower case of a given string */
- def allCaseVersions(s: String, prefix: String = ""): Stream[String] = {
- if (s.isEmpty) {
- Stream(prefix)
- } else {
- allCaseVersions(s.tail, prefix + s.head.toLower) #:::
- allCaseVersions(s.tail, prefix + s.head.toUpper)
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala
deleted file mode 100755
index 538fe04..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import com.thinkaurelius.titan.core.TitanGraph
-import org.apache.hadoop.metadata.query.Expressions._
-import org.slf4j.{LoggerFactory, Logger}
-
-object QueryProcessor {
- val LOG : Logger = LoggerFactory.getLogger("org.apache.hadoop.metadata.query.QueryProcessor")
-
- def evaluate(e: Expression, g: TitanGraph, gP : GraphPersistenceStrategies = GraphPersistenceStrategy1):
- GremlinQueryResult = {
- val e1 = validate(e)
- val q = new GremlinTranslator(e1, gP).translate()
- LOG.debug("Query: " + e1)
- LOG.debug("Expression Tree:\n" + e1.treeString)
- LOG.debug("Gremlin Query: " + q.queryStr)
- new GremlinEvaluator(q, gP, g).evaluate()
- }
-
- def validate(e: Expression): Expression = {
- val e1 = e.transformUp(new Resolver())
-
- e1.traverseUp {
- case x: Expression if !x.resolved =>
- throw new ExpressionException(x, s"Failed to resolved expression $x")
- }
-
- /*
- * trigger computation of dataType of expression tree
- */
- e1.dataType
-
- /*
- * ensure fieldReferences match the input expression's dataType
- */
- val e2 = e1.transformUp(FieldValidator)
- val e3 = e2.transformUp(new Resolver())
-
- e3.dataType
-
- e3
- }
-}
[16/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
new file mode 100755
index 0000000..f6b6a3e
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
@@ -0,0 +1,660 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.commons.lang3.StringUtils;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.text.ParseException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+
+public class DataTypes {
+
+ public static BooleanType BOOLEAN_TYPE = new BooleanType();
+ public static ByteType BYTE_TYPE = new ByteType();
+ public static ShortType SHORT_TYPE = new ShortType();
+ public static IntType INT_TYPE = new IntType();
+ public static LongType LONG_TYPE = new LongType();
+ public static FloatType FLOAT_TYPE = new FloatType();
+ public static DoubleType DOUBLE_TYPE = new DoubleType();
+ public static BigIntegerType BIGINTEGER_TYPE = new BigIntegerType();
+ public static BigDecimalType BIGDECIMAL_TYPE = new BigDecimalType();
+ public static DateType DATE_TYPE = new DateType();
+ public static StringType STRING_TYPE = new StringType();
+ static String ARRAY_TYPE_PREFIX = "array<";
+ static String ARRAY_TYPE_SUFFIX = ">";
+ static String MAP_TYPE_PREFIX = "map<";
+ static String MAP_TYPE_SUFFIX = ">";
+
+ public static String arrayTypeName(String elemTypeName) {
+ return String.format("%s%s%s", ARRAY_TYPE_PREFIX, elemTypeName, ARRAY_TYPE_SUFFIX);
+ }
+
+ public static String arrayTypeName(IDataType elemType) {
+ return arrayTypeName(elemType.getName());
+ }
+
+ public static String mapTypeName(String keyTypeName, String valueTypeName) {
+ return String.format("%s%s,%s%s", MAP_TYPE_PREFIX,
+ keyTypeName, valueTypeName, MAP_TYPE_SUFFIX);
+ }
+
+ public static String mapTypeName(IDataType keyType, IDataType valueType) {
+ return mapTypeName(keyType.getName(), valueType.getName());
+ }
+
+ public static enum TypeCategory {
+ PRIMITIVE,
+ ENUM,
+ ARRAY,
+ MAP,
+ STRUCT,
+ TRAIT,
+ CLASS
+ }
+
+ public static abstract class PrimitiveType<T> extends AbstractDataType<T> {
+ @Override
+ public TypeCategory getTypeCategory() {
+ return TypeCategory.PRIMITIVE;
+ }
+
+ public abstract T nullValue();
+
+ }
+
+ public static class BooleanType extends PrimitiveType<Boolean> {
+
+ private static final String name = "boolean".intern();
+
+ private BooleanType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Boolean convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Boolean) {
+ return (Boolean) val;
+ } else if (val instanceof String) {
+ return Boolean.parseBoolean((String) val);
+ } else if (val instanceof Number) {
+ return ((Number) val).intValue() != 0;
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public Boolean nullValue() {
+ return Boolean.FALSE;
+ }
+ }
+
+ public static class ByteType extends PrimitiveType<Byte> {
+
+ private static final String name = "byte".intern();
+
+ private ByteType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Byte convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Byte) {
+ return (Byte) val;
+ } else if (val instanceof String) {
+ return Byte.parseByte((String) val);
+ } else if (val instanceof Number) {
+ return ((Number) val).byteValue();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public Byte nullValue() {
+ return 0;
+ }
+ }
+
+ public static class ShortType extends PrimitiveType<Short> {
+
+ private static final String name = "short".intern();
+
+ private ShortType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Short convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Short) {
+ return (Short) val;
+ } else if (val instanceof String) {
+ return Short.parseShort((String) val);
+ } else if (val instanceof Number) {
+ return ((Number) val).shortValue();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public Short nullValue() {
+ return 0;
+ }
+ }
+
+ public static class IntType extends PrimitiveType<Integer> {
+
+ private static final String name = "int".intern();
+
+ private IntType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Integer convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Integer) {
+ return (Integer) val;
+ } else if (val instanceof String) {
+ return Integer.parseInt((String) val);
+ } else if (val instanceof Number) {
+ return ((Number) val).intValue();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public Integer nullValue() {
+ return 0;
+ }
+ }
+
+ public static class LongType extends PrimitiveType<Long> {
+
+ private static final String name = "long".intern();
+
+ private LongType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Long convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Long) {
+ return (Long) val;
+ } else if (val instanceof String) {
+ return Long.parseLong((String) val);
+ } else if (val instanceof Number) {
+ return ((Number) val).longValue();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public Long nullValue() {
+ return 0l;
+ }
+ }
+
+ public static class FloatType extends PrimitiveType<Float> {
+
+ private static final String name = "float".intern();
+
+ private FloatType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Float convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Float) {
+ return (Float) val;
+ } else if (val instanceof String) {
+ return Float.parseFloat((String) val);
+ } else if (val instanceof Number) {
+ return ((Number) val).floatValue();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public Float nullValue() {
+ return 0.0f;
+ }
+ }
+
+ public static class DoubleType extends PrimitiveType<Double> {
+
+ private static final String name = "double".intern();
+
+ private DoubleType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Double convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Double) {
+ return (Double) val;
+ } else if (val instanceof String) {
+ return Double.parseDouble((String) val);
+ } else if (val instanceof Number) {
+ return ((Number) val).doubleValue();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public Double nullValue() {
+ return 0.0;
+ }
+ }
+
+ public static class BigIntegerType extends PrimitiveType<BigInteger> {
+
+ private static final String name = "biginteger".intern();
+
+ private BigIntegerType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public BigInteger convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof BigInteger) {
+ return (BigInteger) val;
+ } else if (val instanceof String) {
+ try {
+ return new BigInteger((String) val);
+ } catch (NumberFormatException ne) {
+ throw new ValueConversionException(this, val, ne);
+ }
+ } else if (val instanceof Number) {
+ return BigInteger.valueOf(((Number) val).longValue());
+ } else if (val instanceof BigDecimal) {
+ return ((BigDecimal) val).toBigInteger();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public BigInteger nullValue() {
+ return null;
+ }
+ }
+
+ public static class BigDecimalType extends PrimitiveType<BigDecimal> {
+
+ private static final String name = "bigdecimal".intern();
+
+ private BigDecimalType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public BigDecimal convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof BigDecimal) {
+ return (BigDecimal) val;
+ } else if (val instanceof String) {
+ try {
+ return new BigDecimal((String) val);
+ } catch (NumberFormatException ne) {
+ throw new ValueConversionException(this, val, ne);
+ }
+ } else if (val instanceof Number) {
+ return new BigDecimal(((Number) val).doubleValue());
+ } else if (val instanceof BigInteger) {
+ return new BigDecimal((BigInteger) val);
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ public BigDecimal nullValue() {
+ return null;
+ }
+ }
+
+ public static class DateType extends PrimitiveType<Date> {
+
+ private static final String name = "date".intern();
+
+ private DateType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Date convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof Date) {
+ return (Date) val;
+ } else if (val instanceof String) {
+ try {
+ return TypeSystem.getInstance().getDateFormat().parse((String) val);
+ } catch (ParseException ne) {
+ throw new ValueConversionException(this, val, ne);
+ }
+ } else if (val instanceof Number) {
+ return new Date(((Number) val).longValue());
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ return convertNull(m);
+ }
+
+ @Override
+ public void output(Date val, Appendable buf, String prefix) throws MetadataException {
+ TypeUtils.outputVal(val == null ? "<null>" :
+ TypeSystem.getInstance().getDateFormat().format(val), buf, prefix);
+ }
+
+ public Date nullValue() {
+ return null;
+ }
+ }
+
+ public static class StringType extends PrimitiveType<String> {
+
+ private static final String name = "string".intern();
+
+ private StringType() {
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String convert(Object val, Multiplicity m) throws MetadataException {
+ if (StringUtils.isNotBlank((CharSequence) val)) {
+ return val.toString();
+ }
+ return convertNull(m);
+ }
+
+ public String nullValue() {
+ return null;
+ }
+ }
+
+ public static class ArrayType extends AbstractDataType<ImmutableCollection<?>> {
+ private final String nm;
+ private IDataType elemType;
+
+ public ArrayType(IDataType elemType) {
+ assert elemType != null;
+ this.elemType = elemType;
+ this.nm = arrayTypeName(elemType);
+ }
+
+ public IDataType getElemType() {
+ return elemType;
+ }
+
+ protected void setElemType(IDataType elemType) {
+ this.elemType = elemType;
+ }
+
+ @Override
+ public String getName() {
+ return nm;
+ }
+
+ @Override
+ public ImmutableCollection<?> convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ Iterator it = null;
+ if (val instanceof Collection) {
+ it = ((Collection) val).iterator();
+ } else if (val instanceof Iterable) {
+ it = ((Iterable) val).iterator();
+ } else if (val instanceof Iterator) {
+ it = (Iterator) val;
+ }
+ if (it != null) {
+ ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder()
+ : ImmutableList.builder();
+ while (it.hasNext()) {
+ b.add(elemType.convert(it.next(),
+ TypeSystem.getInstance().allowNullsInCollections()
+ ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
+ }
+ return m.isUnique ? b.build().asList() : b.build();
+ } else {
+ try {
+ return ImmutableList.of(elemType.convert(val,
+ TypeSystem.getInstance().allowNullsInCollections()
+ ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
+ } catch (Exception e) {
+ throw new ValueConversionException(this, val, e);
+ }
+ }
+ }
+ if (!m.nullAllowed()) {
+ throw new ValueConversionException.NullConversionException(m);
+ }
+ return null;
+ }
+
+ public ImmutableCollection<?> mapIds(ImmutableCollection<?> val, Multiplicity m,
+ Map<Id, Id> transientToNewIds)
+ throws MetadataException {
+
+ if (val == null || elemType.getTypeCategory() != TypeCategory.CLASS) {
+ return val;
+ }
+ ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder()
+ : ImmutableList.builder();
+ Iterator it = val.iterator();
+ while (it.hasNext()) {
+ Object elem = it.next();
+ if (elem instanceof IReferenceableInstance) {
+ Id oldId = ((IReferenceableInstance) elem).getId();
+ Id newId = transientToNewIds.get(oldId);
+ b.add(newId == null ? oldId : newId);
+ } else {
+ b.add(elem);
+ }
+ }
+ return b.build();
+ }
+
+ @Override
+ public TypeCategory getTypeCategory() {
+ return TypeCategory.ARRAY;
+ }
+ }
+
+ public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> {
+
+ private final String nm;
+ private IDataType keyType;
+ private IDataType valueType;
+
+ public MapType(IDataType keyType, IDataType valueType) {
+ assert keyType != null;
+ assert valueType != null;
+ this.keyType = keyType;
+ this.valueType = valueType;
+ this.nm = mapTypeName(keyType, valueType);
+ }
+
+ public IDataType getKeyType() {
+ return keyType;
+ }
+
+ protected void setKeyType(IDataType keyType) {
+ this.keyType = keyType;
+ }
+
+ public IDataType getValueType() {
+ return valueType;
+ }
+
+ protected void setValueType(IDataType valueType) {
+ this.keyType = valueType;
+ }
+
+ @Override
+ public String getName() {
+ return nm;
+ }
+
+ @Override
+ public ImmutableMap<?, ?> convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ Iterator<Map.Entry> it = null;
+ if (Map.class.isAssignableFrom(val.getClass())) {
+ it = ((Map) val).entrySet().iterator();
+ ImmutableMap.Builder b = ImmutableMap.builder();
+ while (it.hasNext()) {
+ Map.Entry e = it.next();
+ b.put(keyType.convert(e.getKey(),
+ TypeSystem.getInstance().allowNullsInCollections()
+ ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED),
+ valueType.convert(e.getValue(),
+ TypeSystem.getInstance().allowNullsInCollections()
+ ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
+ }
+ return b.build();
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ if (!m.nullAllowed()) {
+ throw new ValueConversionException.NullConversionException(m);
+ }
+ return null;
+ }
+
+ public ImmutableMap<?, ?> mapIds(ImmutableMap val, Multiplicity m,
+ Map<Id, Id> transientToNewIds)
+ throws MetadataException {
+
+ if (val == null || (keyType.getTypeCategory() != TypeCategory.CLASS &&
+ valueType.getTypeCategory() != TypeCategory.CLASS)) {
+ return val;
+ }
+ ImmutableMap.Builder b = ImmutableMap.builder();
+ Iterator<Map.Entry> it = val.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry elem = it.next();
+ Object oldKey = elem.getKey();
+ Object oldValue = elem.getValue();
+ Object newKey = oldKey;
+ Object newValue = oldValue;
+
+ if (oldKey instanceof IReferenceableInstance) {
+ Id oldId = ((IReferenceableInstance) oldKey).getId();
+ Id newId = transientToNewIds.get(oldId);
+ newKey = newId == null ? oldId : newId;
+ }
+
+ if (oldValue instanceof IReferenceableInstance) {
+ Id oldId = ((IReferenceableInstance) oldValue).getId();
+ Id newId = transientToNewIds.get(oldId);
+ newValue = newId == null ? oldId : newId;
+ }
+
+ b.put(newKey, newValue);
+ }
+ return b.build();
+ }
+
+ @Override
+ public TypeCategory getTypeCategory() {
+ return TypeCategory.MAP;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/DownCastFieldMapping.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DownCastFieldMapping.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DownCastFieldMapping.java
new file mode 100755
index 0000000..0129b4b
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DownCastFieldMapping.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.persistence.DownCastStructInstance;
+
+public class DownCastFieldMapping {
+
+ public final ImmutableMap<String, String> fieldNameMap;
+
+ protected DownCastFieldMapping(ImmutableMap<String, String> fieldNameMap) {
+ this.fieldNameMap = fieldNameMap;
+ }
+
+ public void set(DownCastStructInstance s, String attrName, Object val)
+ throws MetadataException {
+
+ String mappedNm = fieldNameMap.get(attrName);
+ if (mappedNm == null) {
+ throw new ValueConversionException(s.getTypeName(), val, "Unknown field " + attrName);
+ }
+
+ s.backingInstance.set(mappedNm, val);
+ }
+
+ public Object get(DownCastStructInstance s, String attrName) throws MetadataException {
+
+ String mappedNm = fieldNameMap.get(attrName);
+ if (mappedNm == null) {
+ throw new ValueConversionException(
+ String.format("Unknown field %s for Struct %s", attrName, s.getTypeName()));
+ }
+ return s.backingInstance.get(mappedNm);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
new file mode 100755
index 0000000..299f78c
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import scala.math.BigInt;
+
+public class EnumType extends AbstractDataType<EnumValue> {
+
+ public final TypeSystem typeSystem;
+ public final String name;
+ public final ImmutableMap<String, EnumValue> valueMap;
+ public final ImmutableMap<Integer, EnumValue> ordinalMap;
+
+ protected EnumType(TypeSystem typeSystem, String name, EnumValue... values) {
+ this.typeSystem = typeSystem;
+ this.name = name;
+ ImmutableMap.Builder<String, EnumValue> b1 = new ImmutableMap.Builder();
+ ImmutableMap.Builder<Integer, EnumValue> b2 = new ImmutableMap.Builder();
+ for (EnumValue v : values) {
+ b1.put(v.value, v);
+ b2.put(v.ordinal, v);
+ }
+ valueMap = b1.build();
+ ordinalMap = b2.build();
+ }
+
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public EnumValue convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ EnumValue e = null;
+ if (val instanceof EnumValue) {
+ e = valueMap.get(((EnumValue)val).value);
+ } else if ( val instanceof Integer || val instanceof BigInt) {
+ e = ordinalMap.get(val);
+ } else if ( val instanceof String) {
+ e = valueMap.get(val);
+ } else if ( val instanceof Number ) {
+ e = ordinalMap.get(((Number)val).intValue());
+ }
+
+ if (e == null) {
+ throw new ValueConversionException(this, val);
+ }
+ return e;
+ }
+ return convertNull(m);
+ }
+
+ @Override
+ public DataTypes.TypeCategory getTypeCategory() {
+ return DataTypes.TypeCategory.ENUM;
+ }
+
+ public EnumValue fromOrdinal(int o) {
+ return ordinalMap.get(o);
+ }
+
+ public EnumValue fromValue(String val) {
+ return valueMap.get(val.trim());
+ }
+
+ public ImmutableCollection<EnumValue> values() {
+ return valueMap.values();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
new file mode 100755
index 0000000..15be1db
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.ParamChecker;
+
+import java.util.Arrays;
+
+public final class EnumTypeDefinition {
+
+ public final String name;
+ public final EnumValue[] enumValues;
+
+ public EnumTypeDefinition(String name, EnumValue... enumValues) {
+ this.name = ParamChecker.notEmpty(name, "Enum type name");
+ this.enumValues = ParamChecker.notNullElements(enumValues, "Enum values");
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ EnumTypeDefinition that = (EnumTypeDefinition) o;
+
+ if (!Arrays.equals(enumValues, that.enumValues)) return false;
+ if (!name.equals(that.name)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name.hashCode();
+ result = 31 * result + Arrays.hashCode(enumValues);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
new file mode 100755
index 0000000..309257c
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.ParamChecker;
+
+public class EnumValue {
+
+ public final String value;
+ public final int ordinal;
+
+ public EnumValue(String value, int ordinal) {
+ this.value = ParamChecker.notEmpty(value, "Enum value");
+ this.ordinal = ordinal;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ EnumValue enumValue = (EnumValue) o;
+
+ if (ordinal != enumValue.ordinal) return false;
+ if (!value.equals(enumValue.value)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = value.hashCode();
+ result = 31 * result + ordinal;
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/FieldMapping.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/FieldMapping.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/FieldMapping.java
new file mode 100755
index 0000000..dba4600
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/FieldMapping.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.persistence.Id;
+
+import java.util.Map;
+
+public class FieldMapping {
+
+ public final Map<String, AttributeInfo> fields;
+ public final Map<String, Integer> fieldPos;
+ public final Map<String, Integer> fieldNullPos;
+ public final int numBools;
+ public final int numBytes;
+ public final int numShorts;
+ public final int numInts;
+ public final int numLongs;
+ public final int numFloats;
+ public final int numDoubles;
+ public final int numBigInts;
+ public final int numBigDecimals;
+ public final int numDates;
+ public final int numStrings;
+ public final int numArrays;
+ public final int numMaps;
+ public final int numStructs;
+ public final int numReferenceables;
+
+ public FieldMapping(Map<String, AttributeInfo> fields, Map<String, Integer> fieldPos,
+ Map<String, Integer> fieldNullPos, int numBools, int numBytes,
+ int numShorts,
+ int numInts, int numLongs, int numFloats, int numDoubles, int numBigInts,
+ int numBigDecimals,
+ int numDates, int numStrings, int numArrays, int numMaps, int numStructs,
+ int numReferenceables) {
+ this.fields = fields;
+ this.fieldPos = fieldPos;
+ this.fieldNullPos = fieldNullPos;
+ this.numBools = numBools;
+ this.numBytes = numBytes;
+ this.numShorts = numShorts;
+ this.numInts = numInts;
+ this.numLongs = numLongs;
+ this.numFloats = numFloats;
+ this.numDoubles = numDoubles;
+ this.numBigInts = numBigInts;
+ this.numBigDecimals = numBigDecimals;
+ this.numDates = numDates;
+ this.numStrings = numStrings;
+ this.numArrays = numArrays;
+ this.numMaps = numMaps;
+ this.numStructs = numStructs;
+ this.numReferenceables = numReferenceables;
+ }
+
+ protected void outputFields(IStruct s, Appendable buf, String fieldPrefix)
+ throws MetadataException {
+ for (Map.Entry<String, AttributeInfo> e : fields.entrySet()) {
+ String attrName = e.getKey();
+ AttributeInfo i = e.getValue();
+ Object aVal = s.get(attrName);
+ TypeUtils.outputVal(attrName + " : ", buf, fieldPrefix);
+ if (aVal != null && aVal instanceof Id) {
+ TypeUtils.outputVal(aVal.toString(), buf, "");
+ } else {
+ i.dataType().output(aVal, buf, fieldPrefix);
+ }
+ TypeUtils.outputVal("\n", buf, "");
+ }
+ }
+
+ public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
+ if (s == null) {
+ TypeUtils.outputVal("<null>\n", buf, "");
+ return;
+ }
+ TypeUtils.outputVal("{", buf, prefix);
+
+ TypeUtils.outputVal("\n", buf, "");
+ String fieldPrefix = prefix + "\t";
+
+ outputFields(s, buf, fieldPrefix);
+
+ TypeUtils.outputVal("}", buf, prefix);
+ }
+
+ public void output(IReferenceableInstance s, Appendable buf, String prefix)
+ throws MetadataException {
+ if (s == null) {
+ TypeUtils.outputVal("<null>\n", buf, "");
+ return;
+ }
+ TypeUtils.outputVal("{", buf, prefix);
+
+ TypeUtils.outputVal("\n", buf, "");
+ String fieldPrefix = prefix + "\t";
+
+ TypeUtils.outputVal("id : ", buf, fieldPrefix);
+ TypeUtils.outputVal(s.getId().toString(), buf, "");
+ TypeUtils.outputVal("\n", buf, "");
+
+ outputFields(s, buf, fieldPrefix);
+
+ TypeSystem ts = TypeSystem.getInstance();
+
+ for (String sT : s.getTraits()) {
+ TraitType tt = ts.getDataType(TraitType.class, sT);
+ TypeUtils.outputVal(sT + " : ", buf, fieldPrefix);
+ tt.output(s.getTrait(sT), buf, fieldPrefix);
+ }
+
+ TypeUtils.outputVal("}", buf, prefix);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
new file mode 100755
index 0000000..6eb5b16
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
@@ -0,0 +1,499 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.persistence.DownCastStructInstance;
+import org.apache.atlas.typesystem.types.TypeUtils.Pair;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+/**
+ * Represents a Type that can have SuperTypes. An Instance of the HierarchicalType can be
+ * downcast to a SuperType.
+ * @param <ST> the Type of the SuperType. TraitTypes have TraitTypes as SuperTypes, ClassTypes
+ * have ClassTypes
+ * as SuperTypes.
+ * @param <T> the class of the Instance of this DataType.
+ */
+public abstract class HierarchicalType<ST extends HierarchicalType, T> extends AbstractDataType<T>
+ implements Comparable<ST> {
+
+ public final TypeSystem typeSystem;
+ public final Class<ST> superTypeClass;
+ public final String name;
+ public final FieldMapping fieldMapping;
+ public final int numFields;
+ public final ImmutableList<String> superTypes;
+ public final ImmutableList<AttributeInfo> immediateAttrs;
+ public final ImmutableMap<String, String> attributeNameToType;
+ protected ImmutableMap<String, List<Path>> superTypePaths;
+ protected ImmutableMap<String, Path> pathNameToPathMap;
+
+ /**
+ * Used when creating a Type, to support recursive Structs.
+ */
+ HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass,
+ String name, ImmutableList<String> superTypes, int numFields) {
+ this.typeSystem = typeSystem;
+ this.superTypeClass = superTypeClass;
+ this.name = name;
+ this.fieldMapping = null;
+ this.numFields = numFields;
+ this.superTypes = superTypes;
+ this.immediateAttrs = ImmutableList.of();
+ this.attributeNameToType = null;
+ }
+
+ HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass,
+ String name, ImmutableList<String> superTypes, AttributeInfo... fields)
+ throws MetadataException {
+ this.typeSystem = typeSystem;
+ this.superTypeClass = superTypeClass;
+ this.name = name;
+ Pair<FieldMapping, ImmutableMap<String, String>> p = constructFieldMapping(superTypes,
+ fields);
+ this.fieldMapping = p.left;
+ this.attributeNameToType = p.right;
+ this.numFields = this.fieldMapping.fields.size();
+ this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes;
+ this.immediateAttrs = ImmutableList.copyOf(fields);
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ public FieldMapping fieldMapping() {
+ return fieldMapping;
+ }
+
+ /**
+ * Given type must be a SubType of this type.
+ * @param typeName
+ * @throws MetadataException
+ */
+ public boolean isSubType(String typeName) throws MetadataException {
+ HierarchicalType cType = typeSystem.getDataType(HierarchicalType.class, typeName);
+ return (cType == this || cType.superTypePaths.containsKey(getName()));
+ }
+
+ protected void setupSuperTypesGraph()
+ throws MetadataException {
+ setupSuperTypesGraph(superTypes);
+ }
+
+ private void setupSuperTypesGraph(ImmutableList<String> superTypes)
+ throws MetadataException {
+ Map<String, List<Path>> superTypePaths = new HashMap<String, List<Path>>();
+ Map<String, Path> pathNameToPathMap = new HashMap<String, Path>();
+ Queue<Path> queue = new LinkedList<Path>();
+ queue.add(new Node(getName()));
+ while (!queue.isEmpty()) {
+ Path currentPath = queue.poll();
+
+ ST superType = currentPath.typeName == getName() ? (ST) this :
+ (ST) typeSystem.getDataType(superTypeClass, currentPath.typeName);
+
+ pathNameToPathMap.put(currentPath.pathName, currentPath);
+ if (superType != this) {
+ List<Path> typePaths = superTypePaths.get(superType.getName());
+ if (typePaths == null) {
+ typePaths = new ArrayList<Path>();
+ superTypePaths.put(superType.getName(), typePaths);
+ }
+ typePaths.add(currentPath);
+ }
+
+ ImmutableList<String> sTs = superType == this ? superTypes : superType.superTypes;
+
+ if (sTs != null) {
+ for (String sT : sTs) {
+ queue.add(new Path(sT, currentPath));
+ }
+ }
+ }
+
+ this.superTypePaths = ImmutableMap.copyOf(superTypePaths);
+ this.pathNameToPathMap = ImmutableMap.copyOf(pathNameToPathMap);
+
+ }
+
+ protected Pair<FieldMapping, ImmutableMap<String, String>> constructFieldMapping(ImmutableList<String> superTypes,
+ AttributeInfo... fields)
+ throws MetadataException {
+
+ Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
+ Map<String, Integer> fieldPos = new HashMap<String, Integer>();
+ Map<String, Integer> fieldNullPos = new HashMap<String, Integer>();
+ Map<String, String> attributeNameToType = new HashMap<>();
+
+ int numBools = 0;
+ int numBytes = 0;
+ int numShorts = 0;
+ int numInts = 0;
+ int numLongs = 0;
+ int numFloats = 0;
+ int numDoubles = 0;
+ int numBigInts = 0;
+ int numBigDecimals = 0;
+ int numDates = 0;
+ int numStrings = 0;
+ int numArrays = 0;
+ int numMaps = 0;
+ int numStructs = 0;
+ int numReferenceables = 0;
+
+ setupSuperTypesGraph(superTypes);
+
+ Iterator<Path> pathItr = pathIterator();
+ while (pathItr.hasNext()) {
+ Path currentPath = pathItr.next();
+
+ ST superType = currentPath.typeName == getName() ? (ST) this :
+ (ST) typeSystem.getDataType(superTypeClass, currentPath.typeName);
+
+ ImmutableList<AttributeInfo> superTypeFields = superType == this ?
+ ImmutableList.copyOf(fields) : superType.immediateAttrs;
+
+ Set<String> immediateFields = new HashSet<String>();
+
+ for (AttributeInfo i : superTypeFields) {
+ if (superType == this) {
+ if (immediateFields.contains(i.name)) {
+ throw new MetadataException(
+ String.format(
+ "Struct defintion cannot contain multiple fields with the" +
+ " same name %s",
+ i.name));
+ }
+ immediateFields.add(i.name);
+ }
+
+ String attrName = i.name;
+ if (fieldsMap.containsKey(attrName)) {
+ attrName = currentPath.addOverrideAttr(attrName);
+ }
+ attributeNameToType.put(attrName, superType.getName());
+
+ fieldsMap.put(attrName, i);
+ fieldNullPos.put(attrName, fieldNullPos.size());
+ if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
+ fieldPos.put(attrName, numBools);
+ numBools++;
+ } else if (i.dataType() == DataTypes.BYTE_TYPE) {
+ fieldPos.put(attrName, numBytes);
+ numBytes++;
+ } else if (i.dataType() == DataTypes.SHORT_TYPE) {
+ fieldPos.put(attrName, numShorts);
+ numShorts++;
+ } else if (i.dataType() == DataTypes.INT_TYPE) {
+ fieldPos.put(attrName, numInts);
+ numInts++;
+ } else if (i.dataType() == DataTypes.LONG_TYPE) {
+ fieldPos.put(attrName, numLongs);
+ numLongs++;
+ } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
+ fieldPos.put(attrName, numFloats);
+ numFloats++;
+ } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
+ fieldPos.put(attrName, numDoubles);
+ numDoubles++;
+ } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ fieldPos.put(attrName, numBigInts);
+ numBigInts++;
+ } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ fieldPos.put(attrName, numBigDecimals);
+ numBigDecimals++;
+ } else if (i.dataType() == DataTypes.DATE_TYPE) {
+ fieldPos.put(attrName, numDates);
+ numDates++;
+ } else if (i.dataType() == DataTypes.STRING_TYPE) {
+ fieldPos.put(attrName, numStrings);
+ numStrings++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
+ fieldPos.put(i.name, numInts);
+ numInts++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ fieldPos.put(attrName, numArrays);
+ numArrays++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ fieldPos.put(attrName, numMaps);
+ numMaps++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+ i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ fieldPos.put(attrName, numStructs);
+ numStructs++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ fieldPos.put(attrName, numReferenceables);
+ numReferenceables++;
+ } else {
+ throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
+ }
+ }
+ }
+
+ this.superTypePaths = ImmutableMap.copyOf(superTypePaths);
+ this.pathNameToPathMap = ImmutableMap.copyOf(pathNameToPathMap);
+
+ FieldMapping fm = new FieldMapping(fieldsMap,
+ fieldPos,
+ fieldNullPos,
+ numBools,
+ numBytes,
+ numShorts,
+ numInts,
+ numLongs,
+ numFloats,
+ numDoubles,
+ numBigInts,
+ numBigDecimals,
+ numDates,
+ numStrings,
+ numArrays,
+ numMaps,
+ numStructs,
+ numReferenceables);
+
+ return new Pair(fm, ImmutableMap.copyOf(attributeNameToType));
+ }
+
+ public IStruct castAs(IStruct s, String superTypeName) throws MetadataException {
+
+ if (!superTypePaths.containsKey(superTypeName)) {
+ throw new MetadataException(
+ String.format("Cannot downcast to %s from type %s", superTypeName, getName()));
+ }
+
+ if (s != null) {
+ if (s.getTypeName() != getName()) {
+ throw new MetadataException(
+ String.format("Downcast called on wrong type %s, instance type is %s",
+ getName(), s.getTypeName()));
+ }
+
+ List<Path> pathToSuper = superTypePaths.get(superTypeName);
+ if (pathToSuper.size() > 1) {
+ throw new MetadataException(
+ String.format(
+ "Cannot downcast called to %s, from %s: there are multiple paths " +
+ "to SuperType",
+ superTypeName, getName()));
+ }
+
+ ST superType = (ST) typeSystem.getDataType(superTypeClass, superTypeName);
+ Map<String, String> downCastMap = superType
+ .constructDowncastFieldMap(this, pathToSuper.get(0));
+ return new DownCastStructInstance(superTypeName,
+ new DownCastFieldMapping(ImmutableMap.copyOf(downCastMap)),
+ s);
+ }
+
+ return null;
+ }
+
+ public ST getDefinedType(String attrName) throws MetadataException {
+ if (!attributeNameToType.containsKey(attrName)) {
+ throw new MetadataException(String.format("Unknown attribute %s in type %s", attrName, getName()));
+ }
+ return typeSystem.getDataType(superTypeClass, attributeNameToType.get(attrName));
+ }
+
+ public String getDefinedTypeName(String attrName) throws MetadataException {
+ return getDefinedType(attrName).getName();
+ }
+
+ public String getQualifiedName(String attrName) throws MetadataException {
+ String attrTypeName = getDefinedTypeName(attrName);
+ return attrName.contains(".") ? attrName : String.format("%s.%s", attrTypeName, attrName);
+ }
+
+ protected Map<String, String> constructDowncastFieldMap(ST subType, Path pathToSubType) {
+
+ String pathToSubTypeName = pathToSubType.pathAfterThis;
+ /*
+ * the downcastMap;
+ */
+ Map<String, String> dCMap = new HashMap<String, String>();
+ Iterator<Path> itr = pathIterator();
+ while (itr.hasNext()) {
+ Path p = itr.next();
+ Path pInSubType = (Path) subType.pathNameToPathMap
+ .get(p.pathName + "." + pathToSubTypeName);
+
+ if (pInSubType.hiddenAttributeMap != null) {
+ for (Map.Entry<String, String> e : pInSubType.hiddenAttributeMap.entrySet()) {
+ String mappedInThisType =
+ p.hiddenAttributeMap != null ? p.hiddenAttributeMap.get(e.getKey())
+ : null;
+ if (mappedInThisType == null) {
+ dCMap.put(e.getKey(), e.getValue());
+ } else {
+ dCMap.put(mappedInThisType, e.getValue());
+ }
+ }
+ }
+ }
+ return dCMap;
+ }
+
+ @Override
+ public int compareTo(ST o) {
+ String oName = o.getName();
+ if (superTypes.contains(oName)) {
+ return 1;
+ } else if (o.superTypes.contains(getName())) {
+ return -1;
+ } else {
+ return getName().compareTo(oName);
+ }
+ }
+
+ public Set<String> getAllSuperTypeNames() {
+ return superTypePaths.keySet();
+ }
+
+ public Iterator<Path> pathIterator() {
+ return new PathItr();
+ }
+
+ static class Path {
+ public final String typeName;
+ public final String pathName;
+ public final String pathAfterThis;
+ private final Path subTypePath;
+ /*
+ * name mapping for attributes hidden by a SubType.
+ */
+ Map<String, String> hiddenAttributeMap;
+
+ Path(String typeName, Path childPath) throws MetadataException {
+ this.typeName = typeName;
+ this.subTypePath = childPath;
+ if (childPath.contains(typeName)) {
+ throw new CyclicTypeDefinition(this);
+ }
+ pathName = String.format("%s.%s", typeName, childPath.pathName);
+ pathAfterThis = childPath.pathName;
+ }
+
+ Path(String typeName) {
+ assert getClass() == Node.class;
+ this.typeName = typeName;
+ this.subTypePath = null;
+ pathName = typeName;
+ pathAfterThis = null;
+ }
+
+ public boolean contains(String typeName) {
+ return this.typeName.equals(typeName) ||
+ (subTypePath != null && subTypePath.contains(typeName));
+ }
+
+ public String pathString(String nodeSep) {
+
+ StringBuilder b = new StringBuilder();
+ Path p = this;
+
+ while (p != null) {
+ b.append(p.typeName);
+ p = p.subTypePath;
+ if (p != null) {
+ b.append(nodeSep);
+ }
+ }
+ return b.toString();
+ }
+
+ String addOverrideAttr(String name) {
+ hiddenAttributeMap = hiddenAttributeMap == null ? new HashMap<String, String>()
+ : hiddenAttributeMap;
+ String oName = pathName + "." + name;
+ hiddenAttributeMap.put(name, oName);
+ return oName;
+ }
+ }
+
+ static class Node extends Path {
+ Node(String typeName) {
+ super(typeName);
+ }
+ }
+
+ static class CyclicTypeDefinition extends MetadataException {
+
+ CyclicTypeDefinition(Path p) {
+ super(String.format("Cycle in Type Definition %s", p.pathString(" -> ")));
+ }
+ }
+
+ class PathItr implements Iterator<Path> {
+
+ Queue<Path> pathQueue;
+
+ PathItr() {
+ pathQueue = new LinkedList<Path>();
+ pathQueue.add(pathNameToPathMap.get(getName()));
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !pathQueue.isEmpty();
+ }
+
+ @Override
+ public Path next() {
+ Path p = pathQueue.poll();
+ ST t = null;
+ try {
+ t = (ST) typeSystem.getDataType(superTypeClass, p.typeName);
+ } catch (MetadataException me) {
+ throw new RuntimeException(me);
+ }
+ if (t.superTypes != null) {
+ ImmutableList<String> sTs = t.superTypes;
+ for (String sT : sTs) {
+ String nm = sT + "." + p.pathName;
+ pathQueue.add(pathNameToPathMap.get(nm));
+ }
+ }
+ return p;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java
new file mode 100755
index 0000000..20569f1
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalTypeDefinition.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.classification.InterfaceAudience;
+
+public class HierarchicalTypeDefinition<T extends HierarchicalType> extends StructTypeDefinition {
+
+ public final ImmutableList<String> superTypes;
+ public final String hierarchicalMetaTypeName;
+
+ /**
+ * Used for json deserialization only.
+ * not intended public consumption
+ * @param hierarchicalMetaTypeName
+ * @param typeName
+ * @param superTypes
+ * @param attributeDefinitions
+ * @throws ClassNotFoundException
+ */
+ @InterfaceAudience.Private
+ public HierarchicalTypeDefinition(String hierarchicalMetaTypeName,
+ String typeName, String[] superTypes,
+ AttributeDefinition[] attributeDefinitions)
+ throws ClassNotFoundException {
+ this((Class<T>) Class.forName(hierarchicalMetaTypeName),
+ typeName, ImmutableList.copyOf(superTypes), attributeDefinitions);
+ }
+
+ public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType,
+ String typeName, ImmutableList<String> superTypes,
+ AttributeDefinition[] attributeDefinitions) {
+ super(typeName, false, attributeDefinitions);
+ hierarchicalMetaTypeName = hierarchicalMetaType.getName();
+ this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ if (!super.equals(o)) return false;
+
+ HierarchicalTypeDefinition that = (HierarchicalTypeDefinition) o;
+
+ if (!hierarchicalMetaTypeName.equals(that.hierarchicalMetaTypeName)) return false;
+ if (!superTypes.equals(that.superTypes)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + superTypes.hashCode();
+ result = 31 * result + hierarchicalMetaTypeName.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/IConstructableType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IConstructableType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IConstructableType.java
new file mode 100755
index 0000000..6b9bd4c
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IConstructableType.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.ITypedInstance;
+
+import java.util.List;
+
+public interface IConstructableType<U, T extends ITypedInstance> extends IDataType<U> {
+
+ T createInstance() throws MetadataException;
+
+ FieldMapping fieldMapping();
+
+ List<String> getNames(AttributeInfo info);
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
new file mode 100755
index 0000000..e976362
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.MetadataException;
+
+public interface IDataType<T> {
+ String getName();
+
+ T convert(Object val, Multiplicity m) throws MetadataException;
+
+ DataTypes.TypeCategory getTypeCategory();
+
+ void output(T val, Appendable buf, String prefix) throws MetadataException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
new file mode 100755
index 0000000..210c945
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+public final class Multiplicity {
+
+ public static final Multiplicity OPTIONAL = new Multiplicity(0, 1, false);
+ public static final Multiplicity REQUIRED = new Multiplicity(1, 1, false);
+ public static final Multiplicity COLLECTION = new Multiplicity(1, Integer.MAX_VALUE, false);
+ public static final Multiplicity SET = new Multiplicity(1, Integer.MAX_VALUE, true);
+
+ public final int lower;
+ public final int upper;
+ public final boolean isUnique;
+ public Multiplicity(int lower, int upper, boolean isUnique) {
+ assert lower >= 0;
+ assert upper >= 1;
+ assert upper >= lower;
+ this.lower = lower;
+ this.upper = upper;
+ this.isUnique = isUnique;
+ }
+
+ public boolean nullAllowed() {
+ return lower == 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Multiplicity that = (Multiplicity) o;
+
+ if (isUnique != that.isUnique) return false;
+ if (lower != that.lower) return false;
+ if (upper != that.upper) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = lower;
+ result = 31 * result + upper;
+ result = 31 * result + (isUnique ? 1 : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "Multiplicity{" +
+ "lower=" + lower +
+ ", upper=" + upper +
+ ", isUnique=" + isUnique +
+ '}';
+ }
+
+ public String toJson() throws JSONException {
+ JSONObject json = new JSONObject();
+ json.put("lower", lower);
+ json.put("upper", upper);
+ json.put("isUnique", isUnique);
+ return json.toString();
+ }
+
+ public static Multiplicity fromJson(String jsonStr) throws JSONException {
+ JSONObject json = new JSONObject(jsonStr);
+ return new Multiplicity(json.getInt("lower"), json.getInt("upper"), json.getBoolean("isUnique"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphTraversal.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphTraversal.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphTraversal.java
new file mode 100755
index 0000000..5a302ed
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphTraversal.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.persistence.Id;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+public class ObjectGraphTraversal implements Iterator<ObjectGraphTraversal.InstanceTuple> {
+
+ final Queue<InstanceTuple> queue;
+ final TypeSystem typeSystem;
+ Set<Id> processedIds;
+
+ public ObjectGraphTraversal(TypeSystem typeSystem, IReferenceableInstance start)
+ throws MetadataException {
+ this.typeSystem = typeSystem;
+ queue = new LinkedList<InstanceTuple>();
+ processedIds = new HashSet<Id>();
+ processReferenceableInstance(start);
+ }
+
+ void processValue(IDataType dT, Object val) throws MetadataException {
+ if (val != null) {
+ if (dT.getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ IDataType elemType = ((DataTypes.ArrayType) dT).getElemType();
+ processCollection(elemType, val);
+ } else if (dT.getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ IDataType keyType = ((DataTypes.MapType) dT).getKeyType();
+ IDataType valueType = ((DataTypes.MapType) dT).getKeyType();
+ processMap(keyType, valueType, val);
+ } else if (dT.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+ dT.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ processStruct(val);
+ } else if (dT.getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ processReferenceableInstance(val);
+ }
+ }
+ }
+
+ void processMap(IDataType keyType, IDataType valueType, Object val) throws MetadataException {
+ if (keyType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE &&
+ valueType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
+ return;
+ }
+
+ if (val != null) {
+ Iterator<Map.Entry> it = null;
+ if (Map.class.isAssignableFrom(val.getClass())) {
+ it = ((Map) val).entrySet().iterator();
+ ImmutableMap.Builder b = ImmutableMap.builder();
+ while (it.hasNext()) {
+ Map.Entry e = it.next();
+ processValue(keyType, e.getKey());
+ processValue(valueType, e.getValue());
+ }
+ }
+ }
+ }
+
+ void processCollection(IDataType elemType, Object val) throws MetadataException {
+
+ if (elemType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
+ return;
+ }
+
+ if (val != null) {
+ Iterator it = null;
+ if (val instanceof Collection) {
+ it = ((Collection) val).iterator();
+ } else if (val instanceof Iterable) {
+ it = ((Iterable) val).iterator();
+ } else if (val instanceof Iterator) {
+ it = (Iterator) val;
+ }
+ if (it != null) {
+ DataTypes.TypeCategory elemCategory = elemType.getTypeCategory();
+ while (it.hasNext()) {
+ Object elem = it.next();
+ processValue(elemType, elem);
+ }
+ }
+ }
+ }
+
+ void processStruct(Object val) throws MetadataException {
+
+ if (val == null || !(val instanceof IStruct)) {
+ return;
+ }
+
+ IStruct i = (IStruct) val;
+
+ IConstructableType type = typeSystem.getDataType(IConstructableType.class, i.getTypeName());
+
+ for (Map.Entry<String, AttributeInfo> e : type.fieldMapping().fields.entrySet()) {
+ AttributeInfo aInfo = e.getValue();
+ String attrName = e.getKey();
+ if (aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE) {
+ processValue(aInfo.dataType(), i.get(attrName));
+ }
+ }
+ }
+
+ void processReferenceableInstance(Object val) throws MetadataException {
+
+ if (val == null || !(val instanceof IReferenceableInstance || val instanceof Id)) {
+ return;
+ }
+
+ if (val instanceof Id) {
+ Id id = (Id) val;
+ if (id.isUnassigned()) {
+ add(id, null);
+ }
+ return;
+ }
+
+ IReferenceableInstance ref = (IReferenceableInstance) val;
+ Id id = ref.getId();
+ if (id.isUnassigned()) {
+ add(id, ref);
+ if (!processedIds.contains(id)) {
+ processedIds.add(id);
+ processStruct(val);
+
+ ImmutableList<String> traits = ref.getTraits();
+ for (String trait : traits) {
+ processStruct(ref.getTrait(trait));
+ }
+ }
+ }
+ }
+
+ void add(Id id, IReferenceableInstance ref) {
+ queue.add(new InstanceTuple(id, ref));
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ return !queue.isEmpty();
+ }
+
+ @Override
+ public InstanceTuple next() {
+ try {
+ InstanceTuple t = queue.poll();
+ processReferenceableInstance(t.instance);
+ return t;
+ } catch (MetadataException me) {
+ throw new RuntimeException(me);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static class InstanceTuple {
+ public final Id id;
+ public final IReferenceableInstance instance;
+
+ public InstanceTuple(Id id, IReferenceableInstance instance) {
+ this.id = id;
+ this.instance = instance;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
new file mode 100755
index 0000000..7d51ae2
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.persistence.Id;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+/**
+ * Given a IReferenceableInstance, a Walker will traverse the Object Graph
+ * reachable form the instance. It will invoke the process call on the provided NodeProcessor
+ * for each non-primitive attribute (Structs, Traits, References, Arrays of Non-Primitives, Maps
+ * of Non-Primitives)
+ */
+public class ObjectGraphWalker {
+
+ final Queue<IReferenceableInstance> queue;
+ final TypeSystem typeSystem;
+ final NodeProcessor nodeProcessor;
+ Set<Id> processedIds;
+
+ public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor)
+ throws MetadataException {
+ this(typeSystem, nodeProcessor, (IReferenceableInstance) null);
+ }
+
+ public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor,
+ IReferenceableInstance start)
+ throws MetadataException {
+ this.typeSystem = typeSystem;
+ this.nodeProcessor = nodeProcessor;
+ queue = new LinkedList<IReferenceableInstance>();
+ processedIds = new HashSet<Id>();
+ if (start != null) {
+ visitReferenceableInstance(start);
+ }
+ }
+
+ public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor,
+ List<? extends IReferenceableInstance> roots)
+ throws MetadataException {
+ this.typeSystem = typeSystem;
+ this.nodeProcessor = nodeProcessor;
+ queue = new LinkedList<IReferenceableInstance>();
+ processedIds = new HashSet<Id>();
+ for (IReferenceableInstance r : roots) {
+ visitReferenceableInstance(r);
+ }
+ }
+
+ public void walk() throws MetadataException {
+ while (!queue.isEmpty()) {
+ IReferenceableInstance r = queue.poll();
+ processReferenceableInstance(r);
+ }
+ }
+
+ public void addRoot(IReferenceableInstance root) {
+ visitReferenceableInstance(root);
+ }
+
+ void traverseValue(IDataType dT, Object val) throws MetadataException {
+ if (val != null) {
+ if (dT.getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ IDataType elemType = ((DataTypes.ArrayType) dT).getElemType();
+ visitCollection(elemType, val);
+ } else if (dT.getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ IDataType keyType = ((DataTypes.MapType) dT).getKeyType();
+ IDataType valueType = ((DataTypes.MapType) dT).getKeyType();
+ visitMap(keyType, valueType, val);
+ } else if (dT.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+ dT.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ visitStruct(val);
+ } else if (dT.getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ visitReferenceableInstance(val);
+ }
+ }
+ }
+
+ void visitMap(IDataType keyType, IDataType valueType, Object val) throws MetadataException {
+ if (keyType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE &&
+ valueType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
+ return;
+ }
+
+ if (val != null) {
+ Iterator<Map.Entry> it = null;
+ if (Map.class.isAssignableFrom(val.getClass())) {
+ it = ((Map) val).entrySet().iterator();
+ ImmutableMap.Builder b = ImmutableMap.builder();
+ while (it.hasNext()) {
+ Map.Entry e = it.next();
+ traverseValue(keyType, e.getKey());
+ traverseValue(valueType, e.getValue());
+ }
+ }
+ }
+ }
+
+ void visitCollection(IDataType elemType, Object val) throws MetadataException {
+
+ if (elemType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
+ return;
+ }
+
+ if (val != null) {
+ Iterator it = null;
+ if (val instanceof Collection) {
+ it = ((Collection) val).iterator();
+ } else if (val instanceof Iterable) {
+ it = ((Iterable) val).iterator();
+ } else if (val instanceof Iterator) {
+ it = (Iterator) val;
+ }
+ if (it != null) {
+ DataTypes.TypeCategory elemCategory = elemType.getTypeCategory();
+ while (it.hasNext()) {
+ Object elem = it.next();
+ traverseValue(elemType, elem);
+ }
+ }
+ }
+ }
+
+ void visitStruct(Object val) throws MetadataException {
+
+ if (val == null || !(val instanceof IStruct)) {
+ return;
+ }
+
+ IStruct i = (IStruct) val;
+
+ IConstructableType type = typeSystem.getDataType(IConstructableType.class, i.getTypeName());
+
+ for (Map.Entry<String, AttributeInfo> e : type.fieldMapping().fields.entrySet()) {
+ AttributeInfo aInfo = e.getValue();
+ String attrName = e.getKey();
+ if (aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE) {
+ Object aVal = i.get(attrName);
+ nodeProcessor.processNode(new Node(i, attrName, aInfo, aVal));
+ traverseValue(aInfo.dataType(), aVal);
+ }
+ }
+ }
+
+ void visitReferenceableInstance(Object val) {
+
+ if (val == null || !(val instanceof IReferenceableInstance)) {
+ return;
+ }
+
+ IReferenceableInstance ref = (IReferenceableInstance) val;
+
+ if (!processedIds.contains(ref.getId())) {
+ processedIds.add(ref.getId());
+ if (!(ref instanceof Id)) {
+ queue.add(ref);
+ }
+ }
+ }
+
+ void processReferenceableInstance(IReferenceableInstance ref) throws MetadataException {
+
+ nodeProcessor.processNode(new Node(ref, null, null, null));
+ visitStruct(ref);
+ ImmutableList<String> traits = ref.getTraits();
+ for (String trait : traits) {
+ visitStruct(ref.getTrait(trait));
+ }
+ }
+
+ public static interface NodeProcessor {
+
+ void processNode(Node nd) throws MetadataException;
+ }
+
+ /**
+ * Represents a non-primitive value of an instance.
+ */
+ public static class Node {
+ public final IStruct instance;
+ public final String attributeName;
+ public final AttributeInfo aInfo;
+ public final Object value;
+
+ public Node(IStruct instance, String attributeName, AttributeInfo aInfo, Object value) {
+ this.instance = instance;
+ this.attributeName = attributeName;
+ this.aInfo = aInfo;
+ this.value = value;
+ }
+ }
+}
[46/58] [abbrv] incubator-atlas git commit: script modifications to
atlas system props
Posted by ve...@apache.org.
script modifications to atlas system props
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e9dacce0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e9dacce0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e9dacce0
Branch: refs/heads/master
Commit: e9dacce053e6820faf92a2cb2b4f2450505d3845
Parents: fc19063
Author: Jon Maron <jm...@hortonworks.com>
Authored: Wed Jun 10 13:54:04 2015 -0700
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Wed Jun 10 13:54:32 2015 -0700
----------------------------------------------------------------------
src/bin/atlas_start.py | 4 ++--
src/test/python/scripts/TestMetadata.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e9dacce0/src/bin/atlas_start.py
----------------------------------------------------------------------
diff --git a/src/bin/atlas_start.py b/src/bin/atlas_start.py
index f080142..66e8735 100755
--- a/src/bin/atlas_start.py
+++ b/src/bin/atlas_start.py
@@ -22,8 +22,8 @@ import traceback
import atlas_config as mc
METADATA_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=application.log"
-METADATA_COMMAND_OPTS="-Dmetadata.home=%s"
-METADATA_CONFIG_OPTS="-Dmetadata.conf=%s"
+METADATA_COMMAND_OPTS="-Datlas.home=%s"
+METADATA_CONFIG_OPTS="-Datlas.conf=%s"
DEFAULT_JVM_OPTS="-Xmx1024m"
def main():
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e9dacce0/src/test/python/scripts/TestMetadata.py
----------------------------------------------------------------------
diff --git a/src/test/python/scripts/TestMetadata.py b/src/test/python/scripts/TestMetadata.py
index abcd06a..b24b1c1 100644
--- a/src/test/python/scripts/TestMetadata.py
+++ b/src/test/python/scripts/TestMetadata.py
@@ -50,13 +50,13 @@ class TestMetadata(unittest.TestCase):
'org.apache.atlas.Main',
['-app', 'metadata_home/server/webapp/atlas'],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib\\*:metadata_home/libext\\*',
- ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Dmetadata.home=metadata_home', '-Dmetadata.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
+ ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-0Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
else:
java_mock.assert_called_with(
'org.apache.atlas.Main',
['-app', 'metadata_home/server/webapp/atlas'],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib/*:metadata_home/libext/*',
- ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Dmetadata.home=metadata_home', '-Dmetadata.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
+ ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
pass
[33/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
new file mode 100755
index 0000000..07ba7f3
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -0,0 +1,427 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.services;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.GraphTransaction;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.ParamChecker;
+import org.apache.atlas.classification.InterfaceAudience;
+import org.apache.atlas.discovery.SearchIndexer;
+import org.apache.atlas.listener.EntityChangeListener;
+import org.apache.atlas.listener.TypesChangeListener;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.typestore.ITypeStore;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Simple wrapper over TypeSystem and MetadataRepository services with hooks
+ * for listening to changes to the repository.
+ */
+@Singleton
+public class DefaultMetadataService implements MetadataService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultMetadataService.class);
+
+ private final Set<TypesChangeListener> typesChangeListeners = new LinkedHashSet<>();
+ private final Set<EntityChangeListener> entityChangeListeners
+ = new LinkedHashSet<>();
+
+ private final TypeSystem typeSystem;
+ private final MetadataRepository repository;
+ private final ITypeStore typeStore;
+
+ @Inject
+ DefaultMetadataService(MetadataRepository repository,
+ SearchIndexer searchIndexer, ITypeStore typeStore) throws MetadataException {
+ this.typeStore = typeStore;
+ this.typeSystem = TypeSystem.getInstance();
+ this.repository = repository;
+
+ registerListener(searchIndexer);
+ restoreTypeSystem();
+ }
+
+ private void restoreTypeSystem() {
+ LOG.info("Restoring type system from the store");
+ try {
+ TypesDef typesDef = typeStore.restore();
+ typeSystem.defineTypes(typesDef);
+
+ // restore types before creating super types
+ createSuperTypes();
+
+ } catch (MetadataException e) {
+ throw new RuntimeException(e);
+ }
+ LOG.info("Restored type system from the store");
+ }
+
+ private static final AttributeDefinition NAME_ATTRIBUTE =
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE);
+ private static final AttributeDefinition DESCRIPTION_ATTRIBUTE =
+ TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE);
+
+ @InterfaceAudience.Private
+ private void createSuperTypes() throws MetadataException {
+ if (typeSystem.isRegistered(MetadataServiceClient.DATA_SET_SUPER_TYPE)) {
+ return; // this is already registered
+ }
+
+ HierarchicalTypeDefinition<ClassType> infraType =
+ TypesUtil.createClassTypeDef(MetadataServiceClient.INFRASTRUCTURE_SUPER_TYPE,
+ ImmutableList.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE);
+
+ HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
+ .createClassTypeDef(MetadataServiceClient.DATA_SET_SUPER_TYPE,
+ ImmutableList.<String>of(),
+ NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE);
+
+ HierarchicalTypeDefinition<ClassType> processType = TypesUtil
+ .createClassTypeDef(MetadataServiceClient.PROCESS_SUPER_TYPE, ImmutableList.<String>of(),
+ NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE, new AttributeDefinition("inputs",
+ DataTypes.arrayTypeName(MetadataServiceClient.DATA_SET_SUPER_TYPE),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("outputs",
+ DataTypes.arrayTypeName(MetadataServiceClient.DATA_SET_SUPER_TYPE),
+ Multiplicity.OPTIONAL, false, null));
+
+ TypesDef typesDef = TypeUtils
+ .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(infraType, datasetType, processType));
+ createType(TypesSerialization.toJson(typesDef));
+ }
+
+ /**
+ * Creates a new type based on the type system to enable adding
+ * entities (instances for types).
+ *
+ * @param typeDefinition definition as json
+ * @return a unique id for this type
+ */
+ @Override
+ @GraphTransaction
+ public JSONObject createType(String typeDefinition) throws MetadataException {
+ ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty");
+
+ TypesDef typesDef;
+ try {
+ typesDef = TypesSerialization.fromJson(typeDefinition);
+ if(typesDef.isEmpty()) {
+ throw new MetadataException("Invalid type definition");
+ }
+ } catch (Exception e) {
+ LOG.error("Unable to deserialize json={}", typeDefinition, e);
+ throw new IllegalArgumentException("Unable to deserialize json");
+ }
+
+ try {
+ final Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
+
+ try {
+ typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
+ onTypesAddedToRepo(typesAdded);
+ } catch(Throwable t) {
+ typeSystem.removeTypes(ImmutableList.copyOf(typesAdded.keySet()));
+ throw new MetadataException(t);
+ }
+
+ return new JSONObject() {{
+ put(MetadataServiceClient.TYPES, typesAdded.keySet());
+ }};
+ } catch (JSONException e) {
+ LOG.error("Unable to create response for types={}", typeDefinition, e);
+ throw new MetadataException("Unable to create response");
+ }
+ }
+
+ /**
+ * Return the definition for the given type.
+ *
+ * @param typeName name for this type, must be unique
+ * @return type definition as JSON
+ */
+ @Override
+ public String getTypeDefinition(String typeName) throws MetadataException {
+ final IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
+ return TypesSerialization.toJson(typeSystem, dataType.getName());
+ }
+
+ /**
+ * Return the list of types in the repository.
+ *
+ * @return list of type names in the repository
+ */
+ @Override
+ public List<String> getTypeNamesList() throws MetadataException {
+ return typeSystem.getTypeNames();
+ }
+
+ /**
+ * Return the list of trait type names in the type system.
+ *
+ * @return list of trait type names in the type system
+ */
+ @Override
+ public List<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) throws MetadataException {
+ return typeSystem.getTypeNamesByCategory(typeCategory);
+ }
+
+ /**
+ * Creates an entity, instance of the type.
+ *
+ * @param entityInstanceDefinition definition
+ * @return guid
+ */
+ @Override
+ public String createEntity(String entityInstanceDefinition) throws MetadataException {
+ ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition cannot be empty");
+
+ ITypedReferenceableInstance entityTypedInstance =
+ deserializeClassInstance(entityInstanceDefinition);
+
+ final String guid = repository.createEntity(entityTypedInstance);
+
+ onEntityAddedToRepo(entityTypedInstance);
+ return guid;
+ }
+
+ private ITypedReferenceableInstance deserializeClassInstance(
+ String entityInstanceDefinition) throws MetadataException {
+
+ final Referenceable entityInstance;
+ try {
+ entityInstance = InstanceSerialization.fromJsonReferenceable(
+ entityInstanceDefinition, true);
+ } catch (Exception e) { // exception from deserializer
+ LOG.error("Unable to deserialize json={}", entityInstanceDefinition, e);
+ throw new IllegalArgumentException("Unable to deserialize json");
+ }
+ final String entityTypeName = entityInstance.getTypeName();
+ ParamChecker.notEmpty(entityTypeName, "Entity type cannot be null");
+
+ ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
+ return entityType.convert(entityInstance, Multiplicity.REQUIRED);
+ }
+
+ /**
+ * Return the definition for the given guid.
+ *
+ * @param guid guid
+ * @return entity definition as JSON
+ */
+ @Override
+ public String getEntityDefinition(String guid) throws MetadataException {
+ ParamChecker.notEmpty(guid, "guid cannot be null");
+
+ final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
+ return InstanceSerialization.toJson(instance, true);
+ }
+
+ /**
+ * Return the list of entity names for the given type in the repository.
+ *
+ * @param entityType type
+ * @return list of entity names for the given type in the repository
+ */
+ @Override
+ public List<String> getEntityList(String entityType) throws MetadataException {
+ validateTypeExists(entityType);
+
+ return repository.getEntityList(entityType);
+ }
+
+ @Override
+ public void updateEntity(String guid, String property, String value) throws MetadataException {
+ ParamChecker.notEmpty(guid, "guid cannot be null");
+ ParamChecker.notEmpty(property, "property cannot be null");
+ ParamChecker.notEmpty(value, "property value cannot be null");
+
+ repository.updateEntity(guid, property, value);
+ }
+
+ private void validateTypeExists(String entityType) throws MetadataException {
+ ParamChecker.notEmpty(entityType, "entity type cannot be null");
+
+ // verify if the type exists
+ if (!typeSystem.isRegistered(entityType)) {
+ throw new MetadataException("type is not defined for : " + entityType);
+ }
+ }
+
+ /**
+ * Gets the list of trait names for a given entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return a list of trait names for the given entity guid
+ * @throws MetadataException
+ */
+ @Override
+ public List<String> getTraitNames(String guid) throws MetadataException {
+ ParamChecker.notEmpty(guid, "entity GUID cannot be null");
+ return repository.getTraitNames(guid);
+ }
+
+ /**
+ * Adds a new trait to an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitInstanceDefinition trait instance json that needs to be added to entity
+ * @throws MetadataException
+ */
+ @Override
+ public void addTrait(String guid,
+ String traitInstanceDefinition) throws MetadataException {
+ ParamChecker.notEmpty(guid, "entity GUID cannot be null");
+ ParamChecker.notEmpty(traitInstanceDefinition, "Trait instance cannot be null");
+
+ ITypedStruct traitInstance = deserializeTraitInstance(traitInstanceDefinition);
+ final String traitName = traitInstance.getTypeName();
+
+ // ensure trait type is already registered with the TS
+ Preconditions.checkArgument(typeSystem.isRegistered(traitName),
+ "trait=%s should be defined in type system before it can be added", traitName);
+ // ensure trait is not already defined
+ Preconditions.checkArgument(!getTraitNames(guid).contains(traitName),
+ "trait=%s is already defined for entity=%s", traitName, guid);
+
+ repository.addTrait(guid, traitInstance);
+
+ onTraitAddedToEntity(guid, traitName);
+ }
+
+ private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition)
+ throws MetadataException {
+
+ try {
+ Struct traitInstance = InstanceSerialization.fromJsonStruct(
+ traitInstanceDefinition, true);
+ final String entityTypeName = traitInstance.getTypeName();
+ ParamChecker.notEmpty(entityTypeName, "entity type cannot be null");
+
+ TraitType traitType = typeSystem.getDataType(TraitType.class, entityTypeName);
+ return traitType.convert(
+ traitInstance, Multiplicity.REQUIRED);
+ } catch (Exception e) {
+ throw new MetadataException("Error deserializing trait instance", e);
+ }
+ }
+
+ /**
+ * Deletes a given trait from an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitNameToBeDeleted name of the trait
+ * @throws MetadataException
+ */
+ @Override
+ public void deleteTrait(String guid,
+ String traitNameToBeDeleted) throws MetadataException {
+ ParamChecker.notEmpty(guid, "entity GUID cannot be null");
+ ParamChecker.notEmpty(traitNameToBeDeleted, "Trait name cannot be null");
+
+ // ensure trait type is already registered with the TS
+ Preconditions.checkArgument(typeSystem.isRegistered(traitNameToBeDeleted),
+ "trait=%s should be defined in type system before it can be deleted",
+ traitNameToBeDeleted);
+
+ repository.deleteTrait(guid, traitNameToBeDeleted);
+
+ onTraitDeletedFromEntity(guid, traitNameToBeDeleted);
+ }
+
+ private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
+ for (TypesChangeListener listener : typesChangeListeners) {
+ for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
+ listener.onAdd(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ public void registerListener(TypesChangeListener listener) {
+ typesChangeListeners.add(listener);
+ }
+
+ public void unregisterListener(TypesChangeListener listener) {
+ typesChangeListeners.remove(listener);
+ }
+
+ private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance)
+ throws MetadataException {
+
+ for (EntityChangeListener listener : entityChangeListeners) {
+ listener.onEntityAdded(typedInstance);
+ }
+ }
+
+ private void onTraitAddedToEntity(String typeName,
+ String traitName) throws MetadataException {
+ for (EntityChangeListener listener : entityChangeListeners) {
+ listener.onTraitAdded(typeName, traitName);
+ }
+ }
+
+ private void onTraitDeletedFromEntity(String typeName,
+ String traitName) throws MetadataException {
+ for (EntityChangeListener listener : entityChangeListeners) {
+ listener.onTraitDeleted(typeName, traitName);
+ }
+ }
+
+ public void registerListener(EntityChangeListener listener) {
+ entityChangeListeners.add(listener);
+ }
+
+ public void unregisterListener(EntityChangeListener listener) {
+ entityChangeListeners.remove(listener);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/services/MetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/MetadataService.java b/repository/src/main/java/org/apache/atlas/services/MetadataService.java
new file mode 100755
index 0000000..42e86da
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/services/MetadataService.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.services;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+
+/**
+ * Metadata service.
+ */
+public interface MetadataService {
+
+ /**
+ * Creates a new type based on the type system to enable adding
+ * entities (instances for types).
+ *
+ * @param typeDefinition definition as json
+ * @return a unique id for this type
+ */
+ JSONObject createType(String typeDefinition) throws MetadataException;
+
+ /**
+ * Return the definition for the given type.
+ *
+ * @param typeName name for this type, must be unique
+ * @return type definition as JSON
+ */
+ String getTypeDefinition(String typeName) throws MetadataException;
+
+ /**
+ * Return the list of types in the type system.
+ *
+ * @return list of type names in the type system
+ */
+ List<String> getTypeNamesList() throws MetadataException;
+
+ /**
+ * Return the list of trait type names in the type system.
+ *
+ * @return list of trait type names in the type system
+ */
+ List<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) throws MetadataException;
+
+ /**
+ * Creates an entity, instance of the type.
+ *
+ * @param entityDefinition definition
+ * @return guid
+ */
+ String createEntity(String entityDefinition) throws MetadataException;
+
+ /**
+ * Return the definition for the given guid.
+ *
+ * @param guid guid
+ * @return entity definition as JSON
+ */
+ String getEntityDefinition(String guid) throws MetadataException;
+
+ /**
+ * Return the list of entity names for the given type in the repository.
+ *
+ * @param entityType type
+ * @return list of entity names for the given type in the repository
+ */
+ List<String> getEntityList(String entityType) throws MetadataException;
+
+ /**
+ * Adds the property to the given entity id(guid).
+ *
+ * @param guid entity id
+ * @param property property name
+ * @param value property value
+ */
+ void updateEntity(String guid, String property, String value) throws MetadataException;
+
+ // Trait management functions
+ /**
+ * Gets the list of trait names for a given entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return a list of trait names for the given entity guid
+ * @throws MetadataException
+ */
+ List<String> getTraitNames(String guid) throws MetadataException;
+
+ /**
+ * Adds a new trait to an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitInstanceDefinition trait instance that needs to be added to entity
+ * @throws MetadataException
+ */
+ void addTrait(String guid,
+ String traitInstanceDefinition) throws MetadataException;
+
+ /**
+ * Deletes a given trait from an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitNameToBeDeleted name of the trait
+ * @throws MetadataException
+ */
+ void deleteTrait(String guid,
+ String traitNameToBeDeleted) throws MetadataException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/GraphTransaction.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/GraphTransaction.java b/repository/src/main/java/org/apache/hadoop/metadata/GraphTransaction.java
deleted file mode 100644
index e4a099f..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/GraphTransaction.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD)
-public @interface GraphTransaction {}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/GraphTransactionInterceptor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/hadoop/metadata/GraphTransactionInterceptor.java
deleted file mode 100644
index 1e25867..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/GraphTransactionInterceptor.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import com.google.inject.Inject;
-import com.thinkaurelius.titan.core.TitanGraph;
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class GraphTransactionInterceptor implements MethodInterceptor {
- private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionInterceptor.class);
- private TitanGraph titanGraph;
-
- @Inject
- GraphProvider<TitanGraph> graphProvider;
-
- public Object invoke(MethodInvocation invocation) throws Throwable {
- if (titanGraph == null) {
- titanGraph = graphProvider.get();
- }
-
- try {
- Object response = invocation.proceed();
- titanGraph.commit();
- LOG.debug("graph commit");
- return response;
- } catch (Throwable t){
- titanGraph.rollback();
- LOG.debug("graph rollback");
- throw t;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
deleted file mode 100755
index ab870e3..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/RepositoryMetadataModule.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import com.google.inject.Scopes;
-import com.google.inject.matcher.Matchers;
-import com.google.inject.throwingproviders.ThrowingProviderBinder;
-import com.thinkaurelius.titan.core.TitanGraph;
-import org.aopalliance.intercept.MethodInterceptor;
-import org.apache.hadoop.metadata.discovery.DiscoveryService;
-import org.apache.hadoop.metadata.discovery.HiveLineageService;
-import org.apache.hadoop.metadata.discovery.LineageService;
-import org.apache.hadoop.metadata.discovery.SearchIndexer;
-import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
-import org.apache.hadoop.metadata.repository.graph.GraphBackedSearchIndexer;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.apache.hadoop.metadata.repository.graph.TitanGraphProvider;
-import org.apache.hadoop.metadata.repository.typestore.GraphBackedTypeStore;
-import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
-import org.apache.hadoop.metadata.services.DefaultMetadataService;
-import org.apache.hadoop.metadata.services.MetadataService;
-
-/**
- * Guice module for Repository module.
- */
-public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
- @Override
- protected void configure() {
- // special wiring for Titan Graph
- ThrowingProviderBinder.create(binder())
- .bind(GraphProvider.class, TitanGraph.class)
- .to(TitanGraphProvider.class)
- .asEagerSingleton();
-
- // allow for dynamic binding of the metadata repo & graph service
-
- // bind the MetadataRepositoryService interface to an implementation
- bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton();
-
- // bind the ITypeStore interface to an implementation
- bind(ITypeStore.class).to(GraphBackedTypeStore.class).asEagerSingleton();
-
- // bind the GraphService interface to an implementation
- // bind(GraphService.class).to(graphServiceClass);
-
- // bind the MetadataService interface to an implementation
- bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
-
- // bind the DiscoveryService interface to an implementation
- bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
-
- bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class).asEagerSingleton();
-
- bind(LineageService.class).to(HiveLineageService.class).asEagerSingleton();
-
- MethodInterceptor interceptor = new GraphTransactionInterceptor();
- requestInjection(interceptor);
- bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryException.java b/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryException.java
deleted file mode 100755
index c93cac0..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryException.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-import java.security.PrivilegedActionException;
-
-public class DiscoveryException extends MetadataException {
-
- /**
- * Constructs a new exception with the specified detail message. The
- * cause is not initialized, and may subsequently be initialized by
- * a call to {@link #initCause}.
- *
- * @param message the detail message. The detail message is saved for
- * later retrieval by the {@link #getMessage()} method.
- */
- public DiscoveryException(String message) {
- super(message);
- }
-
- /**
- * Constructs a new exception with the specified detail message and
- * cause. <p>Note that the detail message associated with
- * {@code cause} is <i>not</i> automatically incorporated in
- * this exception's detail message.
- *
- * @param message the detail message (which is saved for later retrieval
- * by the {@link #getMessage()} method).
- * @param cause the cause (which is saved for later retrieval by the
- * {@link #getCause()} method). (A <tt>null</tt> value is
- * permitted, and indicates that the cause is nonexistent or
- * unknown.)
- * @since 1.4
- */
- public DiscoveryException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * Constructs a new exception with the specified cause and a detail
- * message of <tt>(cause==null ? null : cause.toString())</tt> (which
- * typically contains the class and detail message of <tt>cause</tt>).
- * This constructor is useful for exceptions that are little more than
- * wrappers for other throwables (for example, {@link
- * PrivilegedActionException}).
- *
- * @param cause the cause (which is saved for later retrieval by the
- * {@link #getCause()} method). (A <tt>null</tt> value is
- * permitted, and indicates that the cause is nonexistent or
- * unknown.)
- * @since 1.4
- */
- public DiscoveryException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryService.java b/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryService.java
deleted file mode 100755
index 0952436..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/discovery/DiscoveryService.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Metadata discovery service.
- */
-public interface DiscoveryService {
-
- /**
- * Full text search
- */
- String searchByFullText(String query) throws DiscoveryException;
-
- /**
- * Search using query DSL.
- *
- * @param dslQuery query in DSL format.
- * @return JSON representing the type and results.
- */
- String searchByDSL(String dslQuery) throws DiscoveryException;
-
- /**
- * Assumes the User is familiar with the persistence structure of the Repository.
- * The given query is run uninterpreted against the underlying Graph Store.
- * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
- *
- * @param gremlinQuery query in gremlin dsl format
- * @return List of Maps
- * @throws org.apache.hadoop.metadata.discovery.DiscoveryException
- */
- List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/discovery/HiveLineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/hadoop/metadata/discovery/HiveLineageService.java
deleted file mode 100644
index 53d042f..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/discovery/HiveLineageService.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery;
-
-import com.thinkaurelius.titan.core.TitanGraph;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.metadata.GraphTransaction;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.ParamChecker;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.apache.hadoop.metadata.discovery.graph.DefaultGraphPersistenceStrategy;
-import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.hadoop.metadata.query.Expressions;
-import org.apache.hadoop.metadata.query.GremlinQueryResult;
-import org.apache.hadoop.metadata.query.HiveLineageQuery;
-import org.apache.hadoop.metadata.query.HiveWhereUsedQuery;
-import org.apache.hadoop.metadata.repository.EntityNotFoundException;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Option;
-import scala.Some;
-import scala.collection.immutable.List;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-/**
- * Hive implementation of Lineage service interface.
- */
-@Singleton
-public class HiveLineageService implements LineageService {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveLineageService.class);
-
- private static final Option<List<String>> SELECT_ATTRIBUTES =
- Some.<List<String>>apply(List.<String>fromArray(new String[]{"name"}));
-
- private static final String HIVE_TABLE_TYPE_NAME;
- private static final String HIVE_PROCESS_TYPE_NAME;
- private static final String HIVE_PROCESS_INPUT_ATTRIBUTE_NAME;
- private static final String HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME;
-
- private static final String HIVE_TABLE_SCHEMA_QUERY;
- private static final String HIVE_TABLE_EXISTS_QUERY;
-
- static {
- // todo - externalize this using type system - dog food
- try {
- PropertiesConfiguration conf = PropertiesUtil.getApplicationProperties();
- HIVE_TABLE_TYPE_NAME =
- conf.getString("metadata.lineage.hive.table.type.name", "DataSet");
- HIVE_PROCESS_TYPE_NAME =
- conf.getString("metadata.lineage.hive.process.type.name", "Process");
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME =
- conf.getString("metadata.lineage.hive.process.inputs.name", "inputs");
- HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME =
- conf.getString("metadata.lineage.hive.process.outputs.name", "outputs");
-
- HIVE_TABLE_SCHEMA_QUERY = conf.getString(
- "metadata.lineage.hive.table.schema.query",
- "hive_table where name=\"%s\", columns");
- HIVE_TABLE_EXISTS_QUERY = conf.getString(
- "metadata.lineage.hive.table.exists.query",
- "from hive_table where name=\"%s\"");
- } catch (MetadataException e) {
- throw new RuntimeException(e);
- }
- }
-
-
- private final TitanGraph titanGraph;
- private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
- private final GraphBackedDiscoveryService discoveryService;
-
- @Inject
- HiveLineageService(GraphProvider<TitanGraph> graphProvider,
- MetadataRepository metadataRepository,
- GraphBackedDiscoveryService discoveryService) throws DiscoveryException {
- this.titanGraph = graphProvider.get();
- this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
- this.discoveryService = discoveryService;
- }
-
- /**
- * Return the lineage outputs for the given tableName.
- *
- * @param tableName tableName
- * @return Lineage Outputs as JSON
- */
- @Override
- @GraphTransaction
- public String getOutputs(String tableName) throws MetadataException {
- LOG.info("Fetching lineage outputs for tableName={}", tableName);
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- validateTableExists(tableName);
-
- HiveWhereUsedQuery outputsQuery = new HiveWhereUsedQuery(
- HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
- Option.empty(), SELECT_ATTRIBUTES, true,
- graphPersistenceStrategy, titanGraph);
-
- Expressions.Expression expression = outputsQuery.expr();
- LOG.debug("Expression is [" + expression.toString() +"]");
- try {
- return discoveryService.evaluate(expression).toJson();
- } catch (Exception e) { // unable to catch ExpressionException
- throw new DiscoveryException("Invalid expression [" + expression.toString() + "]", e);
- }
- }
-
- /**
- * Return the lineage outputs graph for the given tableName.
- *
- * @param tableName tableName
- * @return Outputs Graph as JSON
- */
- @Override
- @GraphTransaction
- public String getOutputsGraph(String tableName) throws MetadataException {
- LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- validateTableExists(tableName);
-
- HiveWhereUsedQuery outputsQuery = new HiveWhereUsedQuery(
- HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
- Option.empty(), SELECT_ATTRIBUTES, true,
- graphPersistenceStrategy, titanGraph);
- return outputsQuery.graph().toInstanceJson();
- }
-
- /**
- * Return the lineage inputs for the given tableName.
- *
- * @param tableName tableName
- * @return Lineage Inputs as JSON
- */
- @Override
- @GraphTransaction
- public String getInputs(String tableName) throws MetadataException {
- LOG.info("Fetching lineage inputs for tableName={}", tableName);
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- validateTableExists(tableName);
-
- HiveLineageQuery inputsQuery = new HiveLineageQuery(
- HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
- Option.empty(), SELECT_ATTRIBUTES, true,
- graphPersistenceStrategy, titanGraph);
-
- Expressions.Expression expression = inputsQuery.expr();
- LOG.debug("Expression is [" + expression.toString() +"]");
- try {
- return discoveryService.evaluate(expression).toJson();
- } catch (Exception e) { // unable to catch ExpressionException
- throw new DiscoveryException("Invalid expression [" + expression.toString() + "]", e);
- }
- }
-
- /**
- * Return the lineage inputs graph for the given tableName.
- *
- * @param tableName tableName
- * @return Inputs Graph as JSON
- */
- @Override
- @GraphTransaction
- public String getInputsGraph(String tableName) throws MetadataException {
- LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- validateTableExists(tableName);
-
- HiveLineageQuery inputsQuery = new HiveLineageQuery(
- HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
- Option.empty(), SELECT_ATTRIBUTES, true,
- graphPersistenceStrategy, titanGraph);
- return inputsQuery.graph().toInstanceJson();
- }
-
- /**
- * Return the schema for the given tableName.
- *
- * @param tableName tableName
- * @return Schema as JSON
- */
- @Override
- @GraphTransaction
- public String getSchema(String tableName) throws MetadataException {
- LOG.info("Fetching schema for tableName={}", tableName);
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- validateTableExists(tableName);
-
- final String schemaQuery = String.format(HIVE_TABLE_SCHEMA_QUERY, tableName);
- return discoveryService.searchByDSL(schemaQuery);
- }
-
- /**
- * Validate if indeed this is a table type and exists.
- *
- * @param tableName table name
- */
- private void validateTableExists(String tableName) throws MetadataException {
- final String tableExistsQuery = String.format(HIVE_TABLE_EXISTS_QUERY, tableName);
- GremlinQueryResult queryResult = discoveryService.evaluate(tableExistsQuery);
- if (!(queryResult.rows().length() > 0)) {
- throw new EntityNotFoundException(tableName + " does not exist");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/discovery/LineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/discovery/LineageService.java b/repository/src/main/java/org/apache/hadoop/metadata/discovery/LineageService.java
deleted file mode 100644
index 1bf2b53..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/discovery/LineageService.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-/**
- * Lineage service interface.
- */
-public interface LineageService {
-
- /**
- * Return the lineage outputs for the given tableName.
- *
- * @param tableName tableName
- * @return Outputs as JSON
- */
- String getOutputs(String tableName) throws MetadataException;
-
- /**
- * Return the lineage outputs graph for the given tableName.
- *
- * @param tableName tableName
- * @return Outputs Graph as JSON
- */
- String getOutputsGraph(String tableName) throws MetadataException;
-
- /**
- * Return the lineage inputs for the given tableName.
- *
- * @param tableName tableName
- * @return Inputs as JSON
- */
- String getInputs(String tableName) throws MetadataException;
-
- /**
- * Return the lineage inputs graph for the given tableName.
- *
- * @param tableName tableName
- * @return Inputs Graph as JSON
- */
- String getInputsGraph(String tableName) throws MetadataException;
-
- /**
- * Return the schema for the given tableName.
- *
- * @param tableName tableName
- * @return Schema as JSON
- */
- String getSchema(String tableName) throws MetadataException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/discovery/SearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/discovery/SearchIndexer.java b/repository/src/main/java/org/apache/hadoop/metadata/discovery/SearchIndexer.java
deleted file mode 100755
index 9921038..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/discovery/SearchIndexer.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery;
-
-import org.apache.hadoop.metadata.listener.TypesChangeListener;
-
-
-/**
- * Interface for indexing types.
- */
-public interface SearchIndexer extends TypesChangeListener {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/DefaultGraphPersistenceStrategy.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/DefaultGraphPersistenceStrategy.java
deleted file mode 100755
index 1e34abd..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/DefaultGraphPersistenceStrategy.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery.graph;
-
-import com.thinkaurelius.titan.core.TitanVertex;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.query.*;
-import org.apache.hadoop.metadata.query.TypeUtils;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.collection.Traversable;
-
-import java.util.List;
-
-/**
- * Default implementation of GraphPersistenceStrategy.
- */
-public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategies {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(DefaultGraphPersistenceStrategy.class);
-
- private final GraphBackedMetadataRepository metadataRepository;
-
- public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) {
- this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository;
- }
-
- @Override
- public String typeAttributeName() {
- return metadataRepository.getTypeAttributeName();
- }
-
- @Override
- public String superTypeAttributeName() {
- return metadataRepository.getSuperTypeAttributeName();
- }
-
- @Override
- public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
- return metadataRepository.getEdgeLabel(dataType, aInfo);
- }
-
- @Override
- public String traitLabel(IDataType<?> dataType, String traitName) {
- return metadataRepository.getTraitLabel(dataType, traitName);
- }
-
- @Override
- public String fieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) {
- try {
- return metadataRepository.getFieldNameInVertex(dataType, aInfo);
- } catch (MetadataException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public List<String> traitNames(TitanVertex vertex) {
- return metadataRepository.getTraitNames(vertex);
- }
-
- @Override
- public String fieldPrefixInSelect() {
- return "it";
- }
-
- @Override
- public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
- return metadataRepository.getIdFromVertex(dataTypeName, vertex);
- }
-
- @Override
- public <U> U constructInstance(IDataType<U> dataType, Object value) {
- try {
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- return dataType.convert(value, Multiplicity.OPTIONAL);
-
- case ARRAY:
- // todo
- break;
-
- case MAP:
- // todo
- break;
-
- case STRUCT:
- TitanVertex structVertex = (TitanVertex) value;
- StructType structType = (StructType) dataType;
- ITypedStruct structInstance = structType.createInstance();
-
- TypeSystem.IdType idType = TypeSystem.getInstance().getIdType();
-
- if (dataType.getName().equals(idType.getName())) {
- structInstance.set(idType.typeNameAttrName(),
- structVertex.getProperty(typeAttributeName()));
- structInstance.set(idType.idAttrName(),
- structVertex.getProperty(idAttributeName()));
-
- } else {
- metadataRepository.getGraphToInstanceMapper().mapVertexToInstance(
- structVertex, structInstance, structType.fieldMapping().fields);
- }
- return dataType.convert(structInstance, Multiplicity.OPTIONAL);
-
- case TRAIT:
- TitanVertex traitVertex = (TitanVertex) value;
- TraitType traitType = (TraitType) dataType;
- ITypedStruct traitInstance = traitType.createInstance();
- // todo - this is not right, we should load the Instance associated with this
- // trait. for now just loading the trait struct.
- // metadataRepository.getGraphToInstanceMapper().mapVertexToTraitInstance(
- // traitVertex, dataType.getName(), , traitType, traitInstance);
- metadataRepository.getGraphToInstanceMapper().mapVertexToInstance(
- traitVertex, traitInstance, traitType.fieldMapping().fields);
- break;
-
- case CLASS:
- TitanVertex classVertex = (TitanVertex) value;
- ITypedReferenceableInstance classInstance =
- metadataRepository.getGraphToInstanceMapper().mapGraphToTypedInstance(
- classVertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
- classVertex);
- return dataType.convert(classInstance, Multiplicity.OPTIONAL);
-
- default:
- throw new UnsupportedOperationException(
- "Load for type " + dataType + "is not supported");
- }
- } catch (MetadataException e) {
- LOG.error("error while constructing an instance", e);
- }
-
- return null;
- }
-
- @Override
- public String edgeLabel(TypeUtils.FieldInfo fInfo) {
- return fInfo.reverseDataType() == null
- ? edgeLabel(fInfo.dataType(), fInfo.attrInfo())
- : edgeLabel(fInfo.reverseDataType(), fInfo.attrInfo());
- }
-
- @Override
- public String gremlinCompOp(Expressions.ComparisonExpression op) {
- return GraphPersistenceStrategies$class.gremlinCompOp(this, op);
- }
-
- @Override
- public String loopObjectExpression(IDataType<?> dataType) {
- return GraphPersistenceStrategies$class.loopObjectExpression(this, dataType);
- }
-
- @Override
- public String instanceToTraitEdgeDirection() { return "out"; }
-
- @Override
- public String traitToInstanceEdgeDirection() { return "in"; }
-
- @Override
- public String idAttributeName() { return metadataRepository.getIdAttributeName(); }
-
- @Override
- public scala.collection.Seq<String> typeTestExpression(String typeName, IntSequence intSeq) {
- return GraphPersistenceStrategies$class.typeTestExpression(this, typeName, intSeq);
- }
-
- @Override
- public boolean collectTypeInstancesIntoVar() {
- return GraphPersistenceStrategies$class.collectTypeInstancesIntoVar(this);
- }
-
- @Override
- public boolean addGraphVertexPrefix(scala.collection.Traversable<String> preStatements) {
- return GraphPersistenceStrategies$class.addGraphVertexPrefix(this, preStatements);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java b/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
deleted file mode 100755
index 37cb835..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/discovery/graph/GraphBackedDiscoveryService.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery.graph;
-
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanIndexQuery;
-import com.thinkaurelius.titan.core.TitanProperty;
-import com.thinkaurelius.titan.core.TitanVertex;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.hadoop.metadata.GraphTransaction;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.discovery.DiscoveryException;
-import org.apache.hadoop.metadata.discovery.DiscoveryService;
-import org.apache.hadoop.metadata.query.Expressions;
-import org.apache.hadoop.metadata.query.GremlinEvaluator;
-import org.apache.hadoop.metadata.query.GremlinQuery;
-import org.apache.hadoop.metadata.query.GremlinQueryResult;
-import org.apache.hadoop.metadata.query.GremlinTranslator;
-import org.apache.hadoop.metadata.query.QueryParser;
-import org.apache.hadoop.metadata.query.QueryProcessor;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.util.Either;
-import scala.util.parsing.combinator.Parsers;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import javax.script.Bindings;
-import javax.script.ScriptEngine;
-import javax.script.ScriptEngineManager;
-import javax.script.ScriptException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Graph backed implementation of Search.
- */
-@Singleton
-public class GraphBackedDiscoveryService implements DiscoveryService {
-
- private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class);
-
- private final TitanGraph titanGraph;
- private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
-
- public final static String SCORE = "score";
-
- @Inject
- GraphBackedDiscoveryService(GraphProvider<TitanGraph> graphProvider,
- MetadataRepository metadataRepository) throws DiscoveryException {
- this.titanGraph = graphProvider.get();
- this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
- }
-
- //Refer http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query
- //http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query
- // .html#query-string-syntax for query syntax
- @Override
- @GraphTransaction
- public String searchByFullText(String query) throws DiscoveryException {
- String graphQuery = String.format("v.%s:(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, query);
- LOG.debug("Full text query: {}", graphQuery);
- Iterator<TitanIndexQuery.Result<Vertex>> results =
- titanGraph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices().iterator();
- JSONArray response = new JSONArray();
-
- while (results.hasNext()) {
- TitanIndexQuery.Result<Vertex> result = results.next();
- Vertex vertex = result.getElement();
-
- JSONObject row = new JSONObject();
- String guid = vertex.getProperty(Constants.GUID_PROPERTY_KEY);
- if (guid != null) { //Filter non-class entities
- try {
- row.put("guid", guid);
- row.put(MetadataServiceClient.TYPENAME, vertex.<String>getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
- row.put(SCORE, result.getScore());
- } catch (JSONException e) {
- LOG.error("Unable to create response", e);
- throw new DiscoveryException("Unable to create response");
- }
-
- response.put(row);
- }
- }
- return response.toString();
- }
-
- /**
- * Search using query DSL.
- *
- * @param dslQuery query in DSL format.
- * @return JSON representing the type and results.
- */
- @Override
- @GraphTransaction
- public String searchByDSL(String dslQuery) throws DiscoveryException {
- LOG.info("Executing dsl query={}", dslQuery);
- GremlinQueryResult queryResult = evaluate(dslQuery);
- return queryResult.toJson();
- }
-
- public GremlinQueryResult evaluate(String dslQuery) throws DiscoveryException {
- LOG.info("Executing dsl query={}", dslQuery);
- try {
- QueryParser queryParser = new QueryParser();
- Either<Parsers.NoSuccess, Expressions.Expression> either = queryParser.apply(dslQuery);
- if (either.isRight()) {
- Expressions.Expression expression = either.right().get();
- return evaluate(expression);
- } else {
- throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left());
- }
- } catch (Exception e) { // unable to catch ExpressionException
- throw new DiscoveryException("Invalid expression : " + dslQuery, e);
- }
- }
-
- public GremlinQueryResult evaluate(Expressions.Expression expression) {
- Expressions.Expression validatedExpression = QueryProcessor.validate(expression);
- GremlinQuery gremlinQuery =
- new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate();
- LOG.debug("Query = {}", validatedExpression);
- LOG.debug("Expression Tree = {}", validatedExpression.treeString());
- LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr());
- return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, titanGraph).evaluate();
- }
-
- /**
- * Assumes the User is familiar with the persistence structure of the Repository.
- * The given query is run uninterpreted against the underlying Graph Store.
- * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
- *
- * @param gremlinQuery query in gremlin dsl format
- * @return List of Maps
- * @throws org.apache.hadoop.metadata.discovery.DiscoveryException
- */
- @Override
- @GraphTransaction
- public List<Map<String, String>> searchByGremlin(String gremlinQuery)
- throws DiscoveryException {
- LOG.info("Executing gremlin query={}", gremlinQuery);
- ScriptEngineManager manager = new ScriptEngineManager();
- ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
- Bindings bindings = engine.createBindings();
- bindings.put("g", titanGraph);
-
- try {
- Object o = engine.eval(gremlinQuery, bindings);
- return extractResult(o);
- } catch (ScriptException se) {
- throw new DiscoveryException(se);
- }
- }
-
- private List<Map<String, String>> extractResult(Object o) throws DiscoveryException {
- if (!(o instanceof List)) {
- throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
- }
-
- List l = (List) o;
- List<Map<String, String>> result = new ArrayList<>();
- for (Object r : l) {
-
- Map<String, String> oRow = new HashMap<>();
- if (r instanceof Map) {
- @SuppressWarnings("unchecked")
- Map<Object, Object> iRow = (Map) r;
- for (Map.Entry e : iRow.entrySet()) {
- Object k = e.getKey();
- Object v = e.getValue();
- oRow.put(k.toString(), v.toString());
- }
- } else if (r instanceof TitanVertex) {
- Iterable<TitanProperty> ps = ((TitanVertex) r).getProperties();
- for (TitanProperty tP : ps) {
- String pName = tP.getPropertyKey().getName();
- Object pValue = ((TitanVertex) r).getProperty(pName);
- if (pValue != null) {
- oRow.put(pName, pValue.toString());
- }
- }
-
- } else if (r instanceof String) {
- oRow.put("", r.toString());
- } else {
- throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
- }
-
- result.add(oRow);
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java b/repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java
deleted file mode 100755
index 12943cc..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/listener/EntityChangeListener.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.listener;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-
-/**
- * Entity (a Typed instance) change notification listener.
- */
-public interface EntityChangeListener {
-
- /**
- * This is upon adding a new typed instance to the repository.
- *
- * @param typedInstance a typed instance
- * @throws org.apache.hadoop.metadata.MetadataException
- */
- void onEntityAdded(ITypedReferenceableInstance typedInstance) throws MetadataException;
-
- /**
- * This is upon adding a new trait to a typed instance.
- *
- * @param guid globally unique identifier for the entity
- * @param traitName trait name for the instance that needs to be added to entity
- * @throws org.apache.hadoop.metadata.MetadataException
- */
- void onTraitAdded(String guid, String traitName) throws MetadataException;
-
- /**
- * This is upon deleting a trait from a typed instance.
- *
- * @param guid globally unique identifier for the entity
- * @param traitName trait name for the instance that needs to be deleted from entity
- * @throws org.apache.hadoop.metadata.MetadataException
- */
- void onTraitDeleted(String guid, String traitName) throws MetadataException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/listener/TypesChangeListener.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/listener/TypesChangeListener.java b/repository/src/main/java/org/apache/hadoop/metadata/listener/TypesChangeListener.java
deleted file mode 100755
index ad83129..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/listener/TypesChangeListener.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.listener;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-
-/**
- * Types change notification listener.
- */
-public interface TypesChangeListener {
-
- /**
- * This is upon adding a new type to Store.
- *
- * @param typeName type name
- * @param dataType data type
- * @throws MetadataException
- */
- void onAdd(String typeName, IDataType dataType) throws MetadataException;
-
- /**
- * This is upon removing an existing type from the Store.
- *
- * @param typeName type name
- * @throws MetadataException
- */
- // void onRemove(String typeName) throws MetadataException;
-
- // This is upon updating an existing type to the store
- // void onChange() throws MetadataException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/Constants.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/Constants.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/Constants.java
deleted file mode 100755
index c16f5d2..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/Constants.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository;
-
-public final class Constants {
-
- /**
- * Globally Unique identifier property key.
- */
-
- public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
- public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid";
- public static final String GUID_INDEX = "guid_index";
-
- /**
- * Entity type name property key.
- */
- public static final String ENTITY_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
- public static final String ENTITY_TYPE_INDEX = "type_index";
-
- /**
- * Entity type's super types property key.
- */
- public static final String SUPER_TYPES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames";
- public static final String SUPER_TYPES_INDEX = "super_types_index";
-
- /**
- * Full-text for the entity for enabling full-text search.
- */
- //weird issue in TitanDB if __ added to this property key. Not adding it for now
- public static final String ENTITY_TEXT_PROPERTY_KEY = "entityText";
-
- /**
- * Properties for type store graph
- */
- public static final String TYPE_CATEGORY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.category";
- public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type";
- public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
-
- /**
- * Trait names property key and index name.
- */
- public static final String TRAIT_NAMES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "traitNames";
- public static final String TRAIT_NAMES_INDEX = "trait_names_index";
-
- public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
- public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
-
- /**
- * search backing index name.
- */
- public static final String BACKING_INDEX = "search";
-
- /**
- * search backing index name for vertex keys.
- */
- public static final String VERTEX_INDEX = "vertex_index";
-
- /**
- * search backing index name for edge labels.
- */
- public static final String EDGE_INDEX = "edge_index";
-
- public static final String FULLTEXT_INDEX = "fulltext_index";
-
- private Constants() {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/DiscoverInstances.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/DiscoverInstances.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/DiscoverInstances.java
deleted file mode 100755
index c937ece..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/DiscoverInstances.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Graph walker implementation for discovering instances.
- */
-public class DiscoverInstances implements ObjectGraphWalker.NodeProcessor {
-
- public final Map<Id, Id> idToNewIdMap;
- public final Map<Id, IReferenceableInstance> idToInstanceMap;
- final IRepository repository;
-
- public DiscoverInstances(IRepository repository) {
- this.repository = repository;
- idToNewIdMap = new HashMap<>();
- idToInstanceMap = new HashMap<>();
- }
-
- @Override
- public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
-
- IReferenceableInstance ref = null;
- Id id = null;
-
- if (nd.attributeName == null) {
- ref = (IReferenceableInstance) nd.instance;
- id = ref.getId();
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (nd.value != null && (nd.value instanceof Id)) {
- id = (Id) nd.value;
- }
- }
-
- if (id != null) {
- if (id.isUnassigned()) {
- if (!idToNewIdMap.containsKey(id)) {
- idToNewIdMap.put(id, repository.newId(id.className));
- }
- if (ref != null && idToInstanceMap.containsKey(ref)) {
- // Oops
- throw new RepositoryException(
- String.format("Unexpected internal error: Id %s processed again", id));
- }
- if (ref != null) {
- idToInstanceMap.put(id, ref);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/EntityNotFoundException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/EntityNotFoundException.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/EntityNotFoundException.java
deleted file mode 100644
index f41bd42..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/EntityNotFoundException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-/**
- * A simple wrapper for 404.
- */
-public class EntityNotFoundException extends RepositoryException {
- public EntityNotFoundException() {
- }
-
- public EntityNotFoundException(String message) {
- super(message);
- }
-
- public EntityNotFoundException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public EntityNotFoundException(Throwable cause) {
- super(cause);
- }
-
- public EntityNotFoundException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/IRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/IRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/IRepository.java
deleted file mode 100755
index 4339ec5..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/IRepository.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository;
-
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-
-import java.util.List;
-
-/**
- * Metadata Repository interface.
- */
-public interface IRepository {
-
- ITypedReferenceableInstance create(IReferenceableInstance i) throws RepositoryException;
-
- ITypedReferenceableInstance update(ITypedReferenceableInstance i) throws RepositoryException;
-
- void delete(ITypedReferenceableInstance i) throws RepositoryException;
-
- Id newId(String typeName);
-
- ITypedReferenceableInstance get(Id id) throws RepositoryException;
-
- void defineClass(ClassType type) throws RepositoryException;
-
- void defineTrait(TraitType type) throws RepositoryException;
-
- void defineTypes(List<HierarchicalType> types) throws RepositoryException;
-}
[39/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
deleted file mode 100755
index 2a341f4..0000000
--- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/bridge/HiveMetaStoreBridge.java
+++ /dev/null
@@ -1,512 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.bridge;
-
-import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.MetadataServiceException;
-import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator;
-import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-/**
- * A Bridge Utility that imports metadata from the Hive Meta Store
- * and registers then in DGI.
- */
-public class HiveMetaStoreBridge {
- private static final String DEFAULT_DGI_URL = "http://localhost:21000/";
- public static final String HIVE_CLUSTER_NAME = "hive.cluster.name";
- public static final String DEFAULT_CLUSTER_NAME = "primary";
- private final String clusterName;
-
- public static final String DGI_URL_PROPERTY = "hive.hook.dgi.url";
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
-
- private final Hive hiveClient;
- private final MetadataServiceClient metadataServiceClient;
-
- /**
- * Construct a HiveMetaStoreBridge.
- * @param hiveConf hive conf
- */
- public HiveMetaStoreBridge(HiveConf hiveConf) throws Exception {
- clusterName = hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
- hiveClient = Hive.get(hiveConf);
- metadataServiceClient = new MetadataServiceClient(hiveConf.get(DGI_URL_PROPERTY, DEFAULT_DGI_URL));
- }
-
- public MetadataServiceClient getMetadataServiceClient() {
- return metadataServiceClient;
- }
-
- public void importHiveMetadata() throws Exception {
- LOG.info("Importing hive metadata");
- importDatabases();
- }
-
- private void importDatabases() throws Exception {
- List<String> databases = hiveClient.getAllDatabases();
- for (String databaseName : databases) {
- Referenceable dbReference = registerDatabase(databaseName);
-
- importTables(databaseName, dbReference);
- }
- }
-
- public Referenceable registerDatabase(String databaseName) throws Exception {
- Referenceable dbRef = getDatabaseReference(databaseName, clusterName);
- if (dbRef == null) {
- LOG.info("Importing objects from databaseName : " + databaseName);
- Database hiveDB = hiveClient.getDatabase(databaseName);
-
- dbRef = new Referenceable(HiveDataTypes.HIVE_DB.getName());
- dbRef.set(HiveDataModelGenerator.NAME, hiveDB.getName().toLowerCase());
- dbRef.set(HiveDataModelGenerator.CLUSTER_NAME, clusterName);
- dbRef.set("description", hiveDB.getDescription());
- dbRef.set("locationUri", hiveDB.getLocationUri());
- dbRef.set("parameters", hiveDB.getParameters());
- dbRef.set("ownerName", hiveDB.getOwnerName());
- if (hiveDB.getOwnerType() != null) {
- dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
- }
-
- dbRef = createInstance(dbRef);
- } else {
- LOG.info("Database {} is already registered with id {}", databaseName, dbRef.getId().id);
- }
- return dbRef;
- }
-
- public Referenceable createInstance(Referenceable referenceable) throws Exception {
- String typeName = referenceable.getTypeName();
- LOG.debug("creating instance of type " + typeName);
-
- String entityJSON = InstanceSerialization.toJson(referenceable, true);
- LOG.debug("Submitting new entity {} = {}", referenceable.getTypeName(), entityJSON);
- JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
- String guid = jsonObject.getString(MetadataServiceClient.GUID);
- LOG.debug("created instance for type " + typeName + ", guid: " + guid);
-
- return new Referenceable(guid, referenceable.getTypeName(), null);
- }
-
- private void importTables(String databaseName, Referenceable databaseReferenceable) throws Exception {
- List<String> hiveTables = hiveClient.getAllTables(databaseName);
-
- for (String tableName : hiveTables) {
- Referenceable tableReferenceable = registerTable(databaseReferenceable, databaseName, tableName);
-
- // Import Partitions
- Referenceable sdReferenceable = getSDForTable(databaseName, tableName);
- registerPartitions(databaseName, tableName, tableReferenceable, sdReferenceable);
-
- // Import Indexes
- importIndexes(databaseName, tableName, databaseReferenceable, tableReferenceable);
- }
- }
-
- /**
- * Gets reference for the database
- *
- *
- * @param databaseName database Name
- * @param clusterName cluster name
- * @return Reference for database if exists, else null
- * @throws Exception
- */
- private Referenceable getDatabaseReference(String databaseName, String clusterName) throws Exception {
- LOG.debug("Getting reference for database {}", databaseName);
- String typeName = HiveDataTypes.HIVE_DB.getName();
-
- String dslQuery = String.format("%s where %s = '%s' and %s = '%s'", typeName,
- HiveDataModelGenerator.NAME, databaseName.toLowerCase(), HiveDataModelGenerator.CLUSTER_NAME,
- clusterName);
- return getEntityReferenceFromDSL(typeName, dslQuery);
- }
-
- public Referenceable getProcessReference(String queryStr) throws Exception {
- LOG.debug("Getting reference for process with query {}", queryStr);
- String typeName = HiveDataTypes.HIVE_PROCESS.getName();
-
- //todo enable DSL
-// String dslQuery = String.format("%s where queryText = \"%s\"", typeName, queryStr);
-// return getEntityReferenceFromDSL(typeName, dslQuery);
-
- String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.queryText', \"%s\").toList()",
- typeName, typeName, StringEscapeUtils.escapeJava(queryStr));
- return getEntityReferenceFromGremlin(typeName, gremlinQuery);
- }
-
- private Referenceable getEntityReferenceFromDSL(String typeName, String dslQuery) throws Exception {
- MetadataServiceClient dgiClient = getMetadataServiceClient();
- JSONArray results = dgiClient.searchByDSL(dslQuery);
- if (results.length() == 0) {
- return null;
- } else {
- String guid;
- JSONObject row = results.getJSONObject(0);
- if (row.has("$id$")) {
- guid = row.getJSONObject("$id$").getString("id");
- } else {
- guid = row.getJSONObject("_col_0").getString("id");
- }
- return new Referenceable(guid, typeName, null);
- }
- }
-
- public static String getTableName(String clusterName, String dbName, String tableName) {
- return String.format("%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(), clusterName);
- }
-
- /**
- * Gets reference for the table
- *
- * @param dbName database name
- * @param tableName table name
- * @return table reference if exists, else null
- * @throws Exception
- */
- private Referenceable getTableReference(String dbName, String tableName) throws Exception {
- LOG.debug("Getting reference for table {}.{}", dbName, tableName);
-
- String typeName = HiveDataTypes.HIVE_TABLE.getName();
- String entityName = getTableName(clusterName, dbName, tableName);
- String dslQuery = String.format("%s as t where name = '%s'", typeName, entityName);
- return getEntityReferenceFromDSL(typeName, dslQuery);
- }
-
- private Referenceable getEntityReferenceFromGremlin(String typeName, String gremlinQuery) throws MetadataServiceException,
- JSONException {
- MetadataServiceClient client = getMetadataServiceClient();
- JSONObject response = client.searchByGremlin(gremlinQuery);
- JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
- if (results.length() == 0) {
- return null;
- }
- String guid = results.getJSONObject(0).getString("__guid");
- return new Referenceable(guid, typeName, null);
- }
-
- private Referenceable getPartitionReference(String dbName, String tableName, List<String> values) throws Exception {
- String valuesStr = "['" + StringUtils.join(values, "', '") + "']";
- LOG.debug("Getting reference for partition for {}.{} with values {}", dbName, tableName, valuesStr);
- String typeName = HiveDataTypes.HIVE_PARTITION.getName();
-
- //todo replace gremlin with DSL
- // String dslQuery = String.format("%s as p where values = %s, tableName where name = '%s', "
- // + "dbName where name = '%s' and clusterName = '%s' select p", typeName, valuesStr, tableName,
- // dbName, clusterName);
-
- String datasetType = MetadataServiceClient.DATA_SET_SUPER_TYPE;
- String tableEntityName = getTableName(clusterName, dbName, tableName);
-
- String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', %s).as('p')."
- + "out('__%s.table').has('%s.name', '%s').back('p').toList()", typeName, typeName, valuesStr,
- typeName, datasetType, tableEntityName);
-
- return getEntityReferenceFromGremlin(typeName, gremlinQuery);
- }
-
- private Referenceable getSDForTable(String dbName, String tableName) throws Exception {
- Referenceable tableRef = getTableReference(dbName, tableName);
- if (tableRef == null) {
- throw new IllegalArgumentException("Table " + dbName + "." + tableName + " doesn't exist");
- }
-
- MetadataServiceClient dgiClient = getMetadataServiceClient();
- Referenceable tableInstance = dgiClient.getEntity(tableRef.getId().id);
- Id sdId = (Id) tableInstance.get("sd");
- return new Referenceable(sdId.id, sdId.getTypeName(), null);
- }
-
- public Referenceable registerTable(String dbName, String tableName) throws Exception {
- Referenceable dbReferenceable = registerDatabase(dbName);
- return registerTable(dbReferenceable, dbName, tableName);
- }
-
- public Referenceable registerTable(Referenceable dbReference, String dbName, String tableName) throws Exception {
- LOG.info("Attempting to register table [" + tableName + "]");
- Referenceable tableRef = getTableReference(dbName, tableName);
- if (tableRef == null) {
- LOG.info("Importing objects from " + dbName + "." + tableName);
-
- Table hiveTable = hiveClient.getTable(dbName, tableName);
-
- tableRef = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
- tableRef.set(HiveDataModelGenerator.NAME,
- getTableName(clusterName, hiveTable.getDbName(), hiveTable.getTableName()));
- tableRef.set(HiveDataModelGenerator.TABLE_NAME, hiveTable.getTableName().toLowerCase());
- tableRef.set("owner", hiveTable.getOwner());
-
- tableRef.set("createTime", hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME));
- tableRef.set("lastAccessTime", hiveTable.getLastAccessTime());
- tableRef.set("retention", hiveTable.getRetention());
-
- tableRef.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT));
-
- // add reference to the database
- tableRef.set(HiveDataModelGenerator.DB, dbReference);
-
- List<Referenceable> colList = getColumns(hiveTable.getCols());
- tableRef.set("columns", colList);
-
- // add reference to the StorageDescriptor
- StorageDescriptor storageDesc = hiveTable.getSd();
- Referenceable sdReferenceable = fillStorageDescStruct(storageDesc, colList);
- tableRef.set("sd", sdReferenceable);
-
- // add reference to the Partition Keys
- List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys());
- tableRef.set("partitionKeys", partKeys);
-
- tableRef.set("parameters", hiveTable.getParameters());
-
- if (hiveTable.getViewOriginalText() != null) {
- tableRef.set("viewOriginalText", hiveTable.getViewOriginalText());
- }
-
- if (hiveTable.getViewExpandedText() != null) {
- tableRef.set("viewExpandedText", hiveTable.getViewExpandedText());
- }
-
- tableRef.set("tableType", hiveTable.getTableType().name());
- tableRef.set("temporary", hiveTable.isTemporary());
-
-
- tableRef = createInstance(tableRef);
- } else {
- LOG.info("Table {}.{} is already registered with id {}", dbName, tableName, tableRef.getId().id);
- }
- return tableRef;
- }
-
- private void registerPartitions(String db, String tableName, Referenceable tableReferenceable,
- Referenceable sdReferenceable) throws Exception {
- Set<Partition> tableParts = hiveClient.getAllPartitionsOf(new Table(Table.getEmptyTable(db, tableName)));
-
- if (tableParts.size() > 0) {
- for (Partition hivePart : tableParts) {
- registerPartition(hivePart, tableReferenceable, sdReferenceable);
- }
- }
- }
-
- public Referenceable registerPartition(Partition partition) throws Exception {
- String dbName = partition.getTable().getDbName();
- String tableName = partition.getTable().getTableName();
- Referenceable tableRef = registerTable(dbName, tableName);
- Referenceable sdRef = getSDForTable(dbName, tableName);
- return registerPartition(partition, tableRef, sdRef);
- }
-
- private Referenceable registerPartition(Partition hivePart, Referenceable tableReferenceable,
- Referenceable sdReferenceable) throws Exception {
- LOG.info("Registering partition for {} with values {}", tableReferenceable,
- StringUtils.join(hivePart.getValues(), ","));
- String dbName = hivePart.getTable().getDbName();
- String tableName = hivePart.getTable().getTableName();
-
- Referenceable partRef = getPartitionReference(dbName, tableName, hivePart.getValues());
- if (partRef == null) {
- partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.getName());
- partRef.set("values", hivePart.getValues());
-
- partRef.set(HiveDataModelGenerator.TABLE, tableReferenceable);
-
- //todo fix
- partRef.set("createTime", hivePart.getLastAccessTime());
- partRef.set("lastAccessTime", hivePart.getLastAccessTime());
-
- // sdStruct = fillStorageDescStruct(hivePart.getSd());
- // Instead of creating copies of the sdstruct for partitions we are reusing existing
- // ones will fix to identify partitions with differing schema.
- partRef.set("sd", sdReferenceable);
-
- partRef.set("parameters", hivePart.getParameters());
- partRef = createInstance(partRef);
- } else {
- LOG.info("Partition {}.{} with values {} is already registered with id {}", dbName, tableName,
- StringUtils.join(hivePart.getValues(), ","), partRef.getId().id);
- }
- return partRef;
- }
-
- private void importIndexes(String db, String table,
- Referenceable dbReferenceable,
- Referenceable tableReferenceable) throws Exception {
- List<Index> indexes = hiveClient.getIndexes(db, table, Short.MAX_VALUE);
- if (indexes.size() > 0) {
- for (Index index : indexes) {
- importIndex(index, dbReferenceable, tableReferenceable);
- }
- }
- }
-
- //todo should be idempotent
- private void importIndex(Index index,
- Referenceable dbReferenceable,
- Referenceable tableReferenceable) throws Exception {
- LOG.info("Importing index {} for {}.{}", index.getIndexName(), dbReferenceable, tableReferenceable);
- Referenceable indexRef = new Referenceable(HiveDataTypes.HIVE_INDEX.getName());
-
- indexRef.set(HiveDataModelGenerator.NAME, index.getIndexName());
- indexRef.set("indexHandlerClass", index.getIndexHandlerClass());
-
- indexRef.set(HiveDataModelGenerator.DB, dbReferenceable);
-
- indexRef.set("createTime", index.getCreateTime());
- indexRef.set("lastAccessTime", index.getLastAccessTime());
- indexRef.set("origTable", index.getOrigTableName());
- indexRef.set("indexTable", index.getIndexTableName());
-
- Referenceable sdReferenceable = fillStorageDescStruct(index.getSd(), null);
- indexRef.set("sd", sdReferenceable);
-
- indexRef.set("parameters", index.getParameters());
-
- tableReferenceable.set("deferredRebuild", index.isDeferredRebuild());
-
- createInstance(indexRef);
- }
-
- private Referenceable fillStorageDescStruct(StorageDescriptor storageDesc, List<Referenceable> colList) throws Exception {
- LOG.debug("Filling storage descriptor information for " + storageDesc);
-
- Referenceable sdReferenceable = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
-
- SerDeInfo serdeInfo = storageDesc.getSerdeInfo();
- LOG.debug("serdeInfo = " + serdeInfo);
- // SkewedInfo skewedInfo = storageDesc.getSkewedInfo();
-
- String serdeInfoName = HiveDataTypes.HIVE_SERDE.getName();
- Struct serdeInfoStruct = new Struct(serdeInfoName);
-
- serdeInfoStruct.set(HiveDataModelGenerator.NAME, serdeInfo.getName());
- serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
- serdeInfoStruct.set("parameters", serdeInfo.getParameters());
-
- sdReferenceable.set("serdeInfo", serdeInfoStruct);
- sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
- sdReferenceable.set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
-
- //Use the passed column list if not null, ex: use same references for table and SD
- List<FieldSchema> columns = storageDesc.getCols();
- if (columns != null && !columns.isEmpty()) {
- if (colList != null) {
- sdReferenceable.set("cols", colList);
- } else {
- sdReferenceable.set("cols", getColumns(columns));
- }
- }
-
- List<Struct> sortColsStruct = new ArrayList<>();
- for (Order sortcol : storageDesc.getSortCols()) {
- String hiveOrderName = HiveDataTypes.HIVE_ORDER.getName();
- Struct colStruct = new Struct(hiveOrderName);
- colStruct.set("col", sortcol.getCol());
- colStruct.set("order", sortcol.getOrder());
-
- sortColsStruct.add(colStruct);
- }
- if (sortColsStruct.size() > 0) {
- sdReferenceable.set("sortCols", sortColsStruct);
- }
-
- sdReferenceable.set("location", storageDesc.getLocation());
- sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
- sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
- sdReferenceable.set("compressed", storageDesc.isCompressed());
-
- if (storageDesc.getBucketCols().size() > 0) {
- sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
- }
-
- sdReferenceable.set("parameters", storageDesc.getParameters());
- sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
-
- return createInstance(sdReferenceable);
- }
-
- private List<Referenceable> getColumns(List<FieldSchema> schemaList) throws Exception
- {
- List<Referenceable> colList = new ArrayList<>();
- for (FieldSchema fs : schemaList) {
- LOG.debug("Processing field " + fs);
- Referenceable colReferenceable = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName());
- colReferenceable.set(HiveDataModelGenerator.NAME, fs.getName());
- colReferenceable.set("type", fs.getType());
- colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment());
-
- colList.add(createInstance(colReferenceable));
- }
- return colList;
- }
-
- public synchronized void registerHiveDataModel() throws Exception {
- HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
- MetadataServiceClient dgiClient = getMetadataServiceClient();
-
- //Register hive data model if its not already registered
- if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null ) {
- LOG.info("Registering Hive data model");
- dgiClient.createType(dataModelGenerator.getModelAsJson());
- } else {
- LOG.info("Hive data model is already registered!");
- }
- }
-
- public static void main(String[] argv) throws Exception {
- HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf());
- hiveMetaStoreBridge.registerHiveDataModel();
- hiveMetaStoreBridge.importHiveMetadata();
- }
-
- public void updateTable(Referenceable tableReferenceable, Table newTable) throws MetadataServiceException {
- MetadataServiceClient client = getMetadataServiceClient();
- client.updateEntity(tableReferenceable.getId()._getId(), HiveDataModelGenerator.TABLE_NAME,
- newTable.getTableName().toLowerCase());
- client.updateEntity(tableReferenceable.getId()._getId(), HiveDataModelGenerator.NAME,
- getTableName(clusterName, newTable.getDbName(), newTable.getTableName()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
deleted file mode 100755
index abc8ab7..0000000
--- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/hook/HiveHook.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.hook;
-
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.exec.ExplainTask;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.hooks.Entity;
-import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
-import org.apache.hadoop.hive.ql.hooks.HookContext;
-import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.plan.HiveOperation;
-import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
-import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * DgiHook sends lineage information to the DgiSever.
- */
-public class HiveHook implements ExecuteWithHookContext {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveHook.class);
-
- // wait time determines how long we wait before we exit the jvm on
- // shutdown. Pending requests after that will not be sent.
- private static final int WAIT_TIME = 3;
- private static ExecutorService executor;
-
- private static final String MIN_THREADS = "hive.hook.dgi.minThreads";
- private static final String MAX_THREADS = "hive.hook.dgi.maxThreads";
- private static final String KEEP_ALIVE_TIME = "hive.hook.dgi.keepAliveTime";
-
- private static final int minThreadsDefault = 5;
- private static final int maxThreadsDefault = 5;
- private static final long keepAliveTimeDefault = 10;
- private static boolean typesRegistered = false;
-
- static {
- // anything shared should be initialized here and destroyed in the
- // shutdown hook The hook contract is weird in that it creates a
- // boatload of hooks.
-
- // initialize the async facility to process hook calls. We don't
- // want to do this inline since it adds plenty of overhead for the
- // query.
- HiveConf hiveConf = new HiveConf();
- int minThreads = hiveConf.getInt(MIN_THREADS, minThreadsDefault);
- int maxThreads = hiveConf.getInt(MAX_THREADS, maxThreadsDefault);
- long keepAliveTime = hiveConf.getLong(KEEP_ALIVE_TIME, keepAliveTimeDefault);
-
- executor = new ThreadPoolExecutor(minThreads, maxThreads, keepAliveTime, TimeUnit.MILLISECONDS,
- new LinkedBlockingQueue<Runnable>(),
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("DGI Logger %d").build());
-
- try {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- try {
- executor.shutdown();
- executor.awaitTermination(WAIT_TIME, TimeUnit.SECONDS);
- executor = null;
- } catch (InterruptedException ie) {
- LOG.info("Interrupt received in shutdown.");
- }
- // shutdown client
- }
- });
- } catch (IllegalStateException is) {
- LOG.info("Attempting to send msg while shutdown in progress.");
- }
-
- LOG.info("Created DGI Hook");
- }
-
- class HiveEvent {
- public HiveConf conf;
-
- public Set<ReadEntity> inputs;
- public Set<WriteEntity> outputs;
-
- public String user;
- public HiveOperation operation;
- public QueryPlan queryPlan;
- public HookContext.HookType hookType;
- public JSONObject jsonPlan;
- }
-
- @Override
- public void run(final HookContext hookContext) throws Exception {
- if (executor == null) {
- LOG.info("No executor running. Bail.");
- return;
- }
-
- // clone to avoid concurrent access
- final HiveEvent event = new HiveEvent();
- final HiveConf conf = new HiveConf(hookContext.getConf());
- boolean debug = conf.get("hive.hook.dgi.synchronous", "false").equals("true");
-
- event.conf = conf;
- event.inputs = hookContext.getInputs();
- event.outputs = hookContext.getOutputs();
-
- event.user = hookContext.getUserName() == null ? hookContext.getUgi().getUserName() : hookContext.getUserName();
- event.operation = HiveOperation.valueOf(hookContext.getOperationName());
- event.queryPlan = hookContext.getQueryPlan();
- event.hookType = hookContext.getHookType();
-
- event.jsonPlan = getQueryPlan(event);
-
- if (debug) {
- fireAndForget(event);
- } else {
- executor.submit(new Runnable() {
- @Override
- public void run() {
- try {
- fireAndForget(event);
- } catch (Throwable e) {
- LOG.info("DGI hook failed", e);
- }
- }
- });
- }
- }
-
- private void fireAndForget(HiveEvent event) throws Exception {
- assert event.hookType == HookContext.HookType.POST_EXEC_HOOK : "Non-POST_EXEC_HOOK not supported!";
-
- LOG.info("Entered DGI hook for hook type {} operation {}", event.hookType, event.operation);
- HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(event.conf);
-
- if (!typesRegistered) {
- dgiBridge.registerHiveDataModel();
- typesRegistered = true;
- }
-
- switch (event.operation) {
- case CREATEDATABASE:
- handleCreateDB(dgiBridge, event);
- break;
-
- case CREATETABLE:
- handleCreateTable(dgiBridge, event);
- break;
-
- case CREATETABLE_AS_SELECT:
- case CREATEVIEW:
- case LOAD:
- case EXPORT:
- case IMPORT:
- case QUERY:
- registerProcess(dgiBridge, event);
- break;
-
- case ALTERTABLE_RENAME:
- case ALTERVIEW_RENAME:
- renameTable(dgiBridge, event);
- break;
-
- case ALTERVIEW_AS:
- //update inputs/outputs?
- break;
-
- case ALTERTABLE_ADDCOLS:
- case ALTERTABLE_REPLACECOLS:
- case ALTERTABLE_RENAMECOL:
- break;
-
- default:
- }
- }
-
- private void renameTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
- //crappy, no easy of getting new name
- assert event.inputs != null && event.inputs.size() == 1;
- assert event.outputs != null && event.outputs.size() > 0;
-
- Table oldTable = event.inputs.iterator().next().getTable();
- Table newTable = null;
- for (WriteEntity writeEntity : event.outputs) {
- if (writeEntity.getType() == Entity.Type.TABLE) {
- Table table = writeEntity.getTable();
- if (table.getDbName().equals(oldTable.getDbName()) && !table.getTableName()
- .equals(oldTable.getTableName())) {
- newTable = table;
- break;
- }
- }
- }
- if (newTable == null) {
- LOG.warn("Failed to deduct new name for " + event.queryPlan.getQueryStr());
- return;
- }
-
- Referenceable dbReferenceable = dgiBridge.registerDatabase(oldTable.getDbName());
- Referenceable tableReferenceable =
- dgiBridge.registerTable(dbReferenceable, oldTable.getDbName(), oldTable.getTableName());
- LOG.info("Updating entity name {}.{} to {}", oldTable.getDbName(), oldTable.getTableName(),
- newTable.getTableName());
- dgiBridge.updateTable(tableReferenceable, newTable);
- }
-
- private void handleCreateTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
- for (WriteEntity entity : event.outputs) {
- if (entity.getType() == Entity.Type.TABLE) {
-
- Table table = entity.getTable();
- Referenceable dbReferenceable = dgiBridge.registerDatabase(table.getDbName());
- dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName());
- }
- }
- }
-
- private void handleCreateDB(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
- for (WriteEntity entity : event.outputs) {
- if (entity.getType() == Entity.Type.DATABASE) {
- dgiBridge.registerDatabase(entity.getDatabase().getName());
- }
- }
- }
-
- private String normalize(String str) {
- if (StringUtils.isEmpty(str)) {
- return null;
- }
- return str.toLowerCase().trim();
- }
-
- private void registerProcess(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
- Set<ReadEntity> inputs = event.inputs;
- Set<WriteEntity> outputs = event.outputs;
-
- //Even explain CTAS has operation name as CREATETABLE_AS_SELECT
- if (inputs.isEmpty() && outputs.isEmpty()) {
- LOG.info("Explain statement. Skipping...");
- }
-
- if (event.queryPlan == null) {
- LOG.info("Query plan is missing. Skipping...");
- }
-
- String queryId = event.queryPlan.getQueryId();
- String queryStr = normalize(event.queryPlan.getQueryStr());
- long queryStartTime = event.queryPlan.getQueryStartTime();
-
- LOG.debug("Registering CTAS query: {}", queryStr);
- Referenceable processReferenceable = dgiBridge.getProcessReference(queryStr);
- if (processReferenceable == null) {
- processReferenceable = new Referenceable(HiveDataTypes.HIVE_PROCESS.getName());
- processReferenceable.set("name", event.operation.getOperationName());
- processReferenceable.set("startTime", queryStartTime);
- processReferenceable.set("userName", event.user);
-
- List<Referenceable> source = new ArrayList<>();
- for (ReadEntity readEntity : inputs) {
- if (readEntity.getType() == Entity.Type.TABLE) {
- Table table = readEntity.getTable();
- String dbName = table.getDbName();
- source.add(dgiBridge.registerTable(dbName, table.getTableName()));
- }
- if (readEntity.getType() == Entity.Type.PARTITION) {
- dgiBridge.registerPartition(readEntity.getPartition());
- }
- }
- processReferenceable.set("inputs", source);
-
- List<Referenceable> target = new ArrayList<>();
- for (WriteEntity writeEntity : outputs) {
- if (writeEntity.getType() == Entity.Type.TABLE || writeEntity.getType() == Entity.Type.PARTITION) {
- Table table = writeEntity.getTable();
- String dbName = table.getDbName();
- target.add(dgiBridge.registerTable(dbName, table.getTableName()));
- }
- if (writeEntity.getType() == Entity.Type.PARTITION) {
- dgiBridge.registerPartition(writeEntity.getPartition());
- }
- }
- processReferenceable.set("outputs", target);
- processReferenceable.set("queryText", queryStr);
- processReferenceable.set("queryId", queryId);
- processReferenceable.set("queryPlan", event.jsonPlan.toString());
- processReferenceable.set("endTime", System.currentTimeMillis());
-
- //TODO set
- processReferenceable.set("queryGraph", "queryGraph");
- dgiBridge.createInstance(processReferenceable);
- } else {
- LOG.debug("Query {} is already registered", queryStr);
- }
- }
-
-
- private JSONObject getQueryPlan(HiveEvent event) throws Exception {
- try {
- ExplainTask explain = new ExplainTask();
- explain.initialize(event.conf, event.queryPlan, null);
- List<Task<?>> rootTasks = event.queryPlan.getRootTasks();
- return explain.getJSONPlan(null, null, rootTasks, event.queryPlan.getFetchTask(), true, false, false);
- } catch(Exception e) {
- LOG.warn("Failed to get queryplan", e);
- return new JSONObject();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java
deleted file mode 100755
index 96c7d55..0000000
--- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataModelGenerator.java
+++ /dev/null
@@ -1,482 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.model;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumType;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates hive data model for both metastore entities and DDL/DML queries.
- */
-public class HiveDataModelGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class);
-
- private static final DataTypes.MapType STRING_MAP_TYPE =
- new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
-
- private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
- private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
- private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
-
- public static final String COMMENT = "comment";
-
- public static final String STORAGE_NUM_BUCKETS = "numBuckets";
- public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
-
- public static final String NAME = "name";
- public static final String TABLE_NAME = "tableName";
- public static final String CLUSTER_NAME = "clusterName";
- public static final String TABLE = "table";
- public static final String DB = "db";
-
- public HiveDataModelGenerator() {
- classTypeDefinitions = new HashMap<>();
- enumTypeDefinitionMap = new HashMap<>();
- structTypeDefinitionMap = new HashMap<>();
- }
-
- public void createDataModel() throws MetadataException {
- LOG.info("Generating the Hive Data Model....");
-
- // enums
- createHiveObjectTypeEnum();
- createHivePrincipalTypeEnum();
- createResourceTypeEnum();
-
- // structs
- createSerDeStruct();
- //createSkewedInfoStruct();
- createOrderStruct();
- createResourceUriStruct();
- createStorageDescClass();
-
- // classes
- createDBClass();
- createTypeClass();
- createColumnClass();
- createPartitionClass();
- createTableClass();
- createIndexClass();
- createRoleClass();
-
- // DDL/DML Process
- createProcessClass();
- }
-
- public TypesDef getTypesDef() {
- return TypeUtils.getTypesDef(
- getEnumTypeDefinitions(),
- getStructTypeDefinitions(),
- getTraitTypeDefinitions(),
- getClassTypeDefinitions()
- );
- }
-
- public String getDataModelAsJSON() {
- return TypesSerialization.toJson(getTypesDef());
- }
-
- public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
- return ImmutableList.copyOf(enumTypeDefinitionMap.values());
- }
-
- public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
- return ImmutableList.copyOf(structTypeDefinitionMap.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
- return ImmutableList.copyOf(classTypeDefinitions.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
- return ImmutableList.of();
- }
-
- private void createHiveObjectTypeEnum() throws MetadataException {
- EnumValue values[] = {
- new EnumValue("GLOBAL", 1),
- new EnumValue("DATABASE", 2),
- new EnumValue("TABLE", 3),
- new EnumValue("PARTITION", 4),
- new EnumValue("COLUMN", 5),
- };
-
- EnumTypeDefinition definition = new EnumTypeDefinition(
- HiveDataTypes.HIVE_OBJECT_TYPE.getName(), values);
- enumTypeDefinitionMap.put(HiveDataTypes.HIVE_OBJECT_TYPE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_OBJECT_TYPE.getName());
- }
-
- private void createHivePrincipalTypeEnum() throws MetadataException {
- EnumValue values[] = {
- new EnumValue("USER", 1),
- new EnumValue("ROLE", 2),
- new EnumValue("GROUP", 3),
- };
-
- EnumTypeDefinition definition = new EnumTypeDefinition(
- HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values);
-
- enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
- }
-
- private void createResourceTypeEnum() throws MetadataException {
- EnumValue values[] = {
- new EnumValue("JAR", 1),
- new EnumValue("FILE", 2),
- new EnumValue("ARCHIVE", 3),
- };
- EnumTypeDefinition definition = new EnumTypeDefinition(
- HiveDataTypes.HIVE_RESOURCE_TYPE.getName(), values);
- enumTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCE_TYPE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCE_TYPE.getName());
- }
-
- private void createSerDeStruct() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
- StructTypeDefinition definition = new StructTypeDefinition(
- HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions);
- structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
- }
-
- private void createOrderStruct() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("order", DataTypes.INT_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- };
-
- StructTypeDefinition definition = new StructTypeDefinition(
- HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions);
- structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName());
- }
-
- private void createStorageDescClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("cols",
- String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.COLLECTION, false, null),
- new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(STORAGE_NUM_BUCKETS, DataTypes.INT_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("bucketCols",
- String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("sortCols",
- String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- //new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
- // Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(STORAGE_IS_STORED_AS_SUB_DIRS, DataTypes.BOOLEAN_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
-
- HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
- ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null, attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
- }
-
- /** Revisit later after nested array types are handled by the typesystem **/
-
- private void createResourceUriStruct() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("resourceType", HiveDataTypes.HIVE_RESOURCE_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("uri", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- };
- StructTypeDefinition definition = new StructTypeDefinition(
- HiveDataTypes.HIVE_RESOURCEURI.getName(), attributeDefinitions);
- structTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCEURI.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCEURI.getName());
- }
-
- private void createDBClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(CLUSTER_NAME, DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("description", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("locationUri", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(),
- null, attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
- }
-
- private void createTypeClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("type1", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("type2", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("fields", String.format("array<%s>",
- HiveDataTypes.HIVE_COLUMN.getName()), Multiplicity.OPTIONAL, false, null),
- };
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.getName(),
- null, attributeDefinitions);
-
- classTypeDefinitions.put(HiveDataTypes.HIVE_TYPE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_TYPE.getName());
- }
-
- private void createColumnClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(
- ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(),
- null, attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
- }
-
- private void createPartitionClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("columns",
- DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class,
- HiveDataTypes.HIVE_PARTITION.getName(), null, attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.getName());
- }
-
- private void createTableClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(TABLE_NAME, DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("partitionKeys",
- DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("columns",
- DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("tableType", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(),
- ImmutableList.of("DataSet"), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
- }
-
- private void createIndexClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("indexHandlerClass", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("origTable", HiveDataTypes.HIVE_TABLE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("indexTable", HiveDataTypes.HIVE_TABLE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("deferredRebuild", DataTypes.BOOLEAN_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_INDEX.getName(),
- ImmutableList.of(MetadataServiceClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_INDEX.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_INDEX.getName());
- }
-
- private void createRoleClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("roleName", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- };
- HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
- ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null, attributeDefinitions);
-
- classTypeDefinitions.put(HiveDataTypes.HIVE_ROLE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_ROLE.getName());
- }
-
- private void createProcessClass() throws MetadataException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("startTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("endTime", DataTypes.LONG_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),
- };
-
- HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
- ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(),
- ImmutableList.of(MetadataServiceClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
- }
-
- public String getModelAsJson() throws MetadataException {
- createDataModel();
- return getDataModelAsJSON();
- }
-
- public static void main(String[] args) throws Exception {
- HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
- System.out.println("hiveDataModelAsJSON = " + hiveDataModelGenerator.getModelAsJson());
-
- TypesDef typesDef = hiveDataModelGenerator.getTypesDef();
- for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", enumType.name, EnumType.class.getSimpleName(),
- Arrays.toString(enumType.enumValues)));
- }
- for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", structType.typeName, StructType.class.getSimpleName(),
- Arrays.toString(structType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", classType.typeName, ClassType.class.getSimpleName(),
- Arrays.toString(classType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
- Arrays.toString(traitType.attributeDefinitions)));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java b/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java
deleted file mode 100755
index 39c7d7d..0000000
--- a/addons/hive-bridge/src/main/java/org/apache/hadoop/metadata/hive/model/HiveDataTypes.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.model;
-
-/**
- * Hive Data Types for model and bridge.
- */
-public enum HiveDataTypes {
-
- // Enums
- HIVE_OBJECT_TYPE,
- HIVE_PRINCIPAL_TYPE,
- HIVE_RESOURCE_TYPE,
-
- // Structs
- HIVE_SERDE,
- HIVE_ORDER,
- HIVE_RESOURCEURI,
-
- // Classes
- HIVE_DB,
- HIVE_STORAGEDESC,
- HIVE_TABLE,
- HIVE_COLUMN,
- HIVE_PARTITION,
- HIVE_INDEX,
- HIVE_ROLE,
- HIVE_TYPE,
- HIVE_PROCESS,
- // HIVE_VIEW,
- ;
-
- public String getName() {
- return name().toLowerCase();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
new file mode 100644
index 0000000..084e831
--- /dev/null
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.hive.hook;
+
+import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
+import org.apache.atlas.security.BaseSecurityTest;
+import org.apache.atlas.web.service.SecureEmbeddedServer;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.mortbay.jetty.Server;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
+
+/**
+ *
+ */
+public class BaseSSLAndKerberosTest extends BaseSecurityTest {
+ public static final String TESTUSER = "testuser";
+ public static final String TESTPASS = "testpass";
+ protected static final String DGI_URL = "https://localhost:21443/";
+ protected Path jksPath;
+ protected String providerUrl;
+ protected File httpKeytabFile;
+ private File userKeytabFile;
+
+ class TestSecureEmbeddedServer extends SecureEmbeddedServer {
+
+ public TestSecureEmbeddedServer(int port, String path) throws IOException {
+ super(port, path);
+ }
+
+ public Server getServer() {
+ return server;
+ }
+
+ @Override
+ public PropertiesConfiguration getConfiguration() {
+ return super.getConfiguration();
+ }
+ }
+
+ protected void setupCredentials() throws Exception {
+ Configuration conf = new Configuration(false);
+
+ File file = new File(jksPath.toUri().getPath());
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ // create new aliases
+ try {
+
+ char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ KEYSTORE_PASSWORD_KEY, storepass);
+
+ char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ TRUSTSTORE_PASSWORD_KEY, trustpass);
+
+ char[] trustpass2 = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ "ssl.client.truststore.password", trustpass2);
+
+ char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ SERVER_CERT_PASSWORD_KEY, certpass);
+
+ // write out so that it can be found in checks
+ provider.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ public void setupKDCAndPrincipals() throws Exception {
+ // set up the KDC
+ File kdcWorkDir = startKDC();
+
+ userKeytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
+ httpKeytabFile = createKeytab(kdc, kdcWorkDir, "HTTP", "spnego.service.keytab");
+
+ // create a test user principal
+ kdc.createPrincipal(TESTUSER, TESTPASS);
+
+ StringBuilder jaas = new StringBuilder(1024);
+ jaas.append("TestUser {\n" +
+ " com.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true;\n" +
+ "};\n");
+ jaas.append(createJAASEntry("Client", "dgi", userKeytabFile));
+ jaas.append(createJAASEntry("Server", "HTTP", httpKeytabFile));
+
+ File jaasFile = new File(kdcWorkDir, "jaas.txt");
+ FileUtils.write(jaasFile, jaas.toString());
+ bindJVMtoJAASFile(jaasFile);
+ }
+
+ protected String getWarPath() {
+ return String.format("/../../webapp/target/atlas-webapp-%s",
+ System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
+ }
+
+ protected HiveConf getHiveConf() {
+ HiveConf hiveConf = new HiveConf(this.getClass());
+ hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
+ hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
+ System.getProperty("user.dir") + "/target/atlas");
+ hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
+ hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
+ hiveConf.set("hive.hook.dgi.synchronous", "true");
+ return hiveConf;
+ }
+}
[32/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java
deleted file mode 100755
index 310dfd7..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/MetadataRepository.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-
-import java.util.List;
-
-/**
- * An interface for persisting metadata into a blueprints enabled graph db.
- */
-public interface MetadataRepository {
-
- /**
- * Returns the property key used to store entity type name.
- *
- * @return property key used to store entity type name.
- */
- String getTypeAttributeName();
-
- /**
- * Returns the property key used to store super type names.
- *
- * @return property key used to store super type names.
- */
- String getSuperTypeAttributeName();
-
- /**
- * Return the property key used to store a given traitName in the repository.
- *
- * @param dataType data type
- * @param traitName trait name
- * @return property key used to store a given traitName
- */
- String getTraitLabel(IDataType<?> dataType, String traitName);
-
- /**
- * Return the property key used to store a given attribute in the repository.
- *
- * @param dataType data type
- * @param aInfo attribute info
- * @return property key used to store a given attribute
- */
- String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws MetadataException;
-
- /**
- * Return the edge label for a given attribute in the repository.
- *
- * @param dataType data type
- * @param aInfo attribute info
- * @return edge label for a given attribute
- */
- String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
-
- /**
- * Creates an entity definition (instance) corresponding to a given type.
- *
- * @param entity entity (typed instance)
- * @return a globally unique identifier
- * @throws RepositoryException
- */
- String createEntity(IReferenceableInstance entity) throws RepositoryException;
-
- /**
- * Fetch the complete definition of an entity given its GUID.
- *
- * @param guid globally unique identifier for the entity
- * @return entity (typed instance) definition
- * @throws RepositoryException
- */
- ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
-
- /**
- * Gets the list of entities for a given entity type.
- *
- * @param entityType name of a type which is unique
- * @return a list of entity names for the given type
- * @throws RepositoryException
- */
- List<String> getEntityList(String entityType) throws RepositoryException;
-
- /**
- * Deletes an entity definition (instance) corresponding to a given type.
- *
- * @param guid globally unique identifier for the entity
- * @return true if deleted else false
- * @throws RepositoryException
- */
- // boolean deleteEntity(String guid) throws RepositoryException;
-
- /**
- * Updates an entity given its GUID with the attribute name and value.
- *
- * @param guid globally unique identifier for the entity
- * @param attributeName name of the attribute
- * @param attributeValue value of the attribute
- * @return an entity instance with updated state
- * @throws RepositoryException
- */
- //ITypedReferenceableInstance updateEntity(String guid, String attributeName,
- // String attributeValue) throws RepositoryException;
-
-
- // Trait management functions
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- * @throws RepositoryException
- */
- List<String> getTraitNames(String guid) throws MetadataException;
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitInstance trait instance that needs to be added to entity
- * @throws RepositoryException
- */
- void addTrait(String guid,
- ITypedStruct traitInstance) throws RepositoryException;
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitNameToBeDeleted name of the trait
- * @throws RepositoryException
- */
- void deleteTrait(String guid,
- String traitNameToBeDeleted) throws RepositoryException;
-
- /**
- * Adds the property to the entity that corresponds to the GUID
- * @param guid entity id
- * @param property property name
- * @param value property value
- */
- void updateEntity(String guid, String property, String value) throws RepositoryException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/RepositoryException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/RepositoryException.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/RepositoryException.java
deleted file mode 100755
index effe17a..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/RepositoryException.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-/**
- * Base Exception class for Repository API.
- */
-public class RepositoryException extends MetadataException {
-
- public RepositoryException() {
- }
-
- public RepositoryException(String message) {
- super(message);
- }
-
- public RepositoryException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public RepositoryException(Throwable cause) {
- super(cause);
- }
-
- public RepositoryException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
deleted file mode 100755
index 9b546b1..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
+++ /dev/null
@@ -1,1292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.google.common.base.Preconditions;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanProperty;
-import com.thinkaurelius.titan.core.TitanVertex;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.GraphQuery;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metadata.GraphTransaction;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.repository.EntityNotFoundException;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.persistence.MapIds;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * An implementation backed by a Graph database provided
- * as a Graph Service.
- */
-@Singleton
-public class GraphBackedMetadataRepository implements MetadataRepository {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
- private static final String FULL_TEXT_DELIMITER = " ";
-
- private static final String EDGE_LABEL_PREFIX = "__";
-
- private final AtomicInteger ID_SEQ = new AtomicInteger(0);
-
- private final TypedInstanceToGraphMapper instanceToGraphMapper
- = new TypedInstanceToGraphMapper();
- private final GraphToTypedInstanceMapper graphToInstanceMapper
- = new GraphToTypedInstanceMapper();
-
- private final TypeSystem typeSystem;
- private final TitanGraph titanGraph;
-
- @Inject
- public GraphBackedMetadataRepository(
- GraphProvider<TitanGraph> graphProvider) throws MetadataException {
- this.typeSystem = TypeSystem.getInstance();
-
- this.titanGraph = graphProvider.get();
- }
-
- public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
- return graphToInstanceMapper;
- }
-
- @Override
- public String getTypeAttributeName() {
- return Constants.ENTITY_TYPE_PROPERTY_KEY;
- }
-
- /**
- * Returns the property key used to store super type names.
- *
- * @return property key used to store super type names.
- */
- @Override
- public String getSuperTypeAttributeName() {
- return Constants.SUPER_TYPES_PROPERTY_KEY;
- }
-
- public String getIdAttributeName() {
- return Constants.GUID_PROPERTY_KEY;
- }
-
- @Override
- public String getTraitLabel(IDataType<?> dataType, String traitName) {
- return dataType.getName() + "." + traitName;
- }
-
- @Override
- public String getFieldNameInVertex(IDataType<?> dataType,
- AttributeInfo aInfo) throws MetadataException {
- return getQualifiedName(dataType, aInfo.name);
- }
-
- @Override
- public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
- return getEdgeLabel(dataType.getName(), aInfo.name);
- }
-
- public String getEdgeLabel(String typeName, String attrName) {
- return EDGE_LABEL_PREFIX + typeName + "." + attrName;
- }
-
- public String getEdgeLabel(ITypedInstance typedInstance,
- AttributeInfo aInfo) throws MetadataException {
- IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
- return getEdgeLabel(dataType, aInfo);
- }
-
- @Override
- @GraphTransaction
- public String createEntity(IReferenceableInstance typedInstance) throws RepositoryException {
- LOG.info("adding entity={}", typedInstance);
- try {
- return instanceToGraphMapper.mapTypedInstanceToGraph(typedInstance);
- } catch (MetadataException e) {
- throw new RepositoryException(e);
- }
- }
-
- @Override
- @GraphTransaction
- public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException {
- LOG.info("Retrieving entity with guid={}", guid);
-
- Vertex instanceVertex = getVertexForGUID(guid);
-
- try {
- LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
- return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
-
- } catch (MetadataException e) {
- throw new RepositoryException(e);
- }
- }
-
- private Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
- Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid);
- if (instanceVertex == null) {
- LOG.debug("Could not find a vertex for guid={}", guid);
- throw new EntityNotFoundException(
- "Could not find an entity in the repository for guid: " + guid);
- }
-
- return instanceVertex;
- }
-
- @Override
- public List<String> getEntityList(String entityType) throws RepositoryException {
- LOG.info("Retrieving entity list for type={}", entityType);
- GraphQuery query = titanGraph.query()
- .has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
- Iterator<Vertex> results = query.vertices().iterator();
- if (!results.hasNext()) {
- return Collections.emptyList();
- }
-
- ArrayList<String> entityList = new ArrayList<>();
- while (results.hasNext()) {
- Vertex vertex = results.next();
- entityList.add(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY));
- }
-
- return entityList;
- }
-
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- * @throws RepositoryException
- */
- @Override
- @GraphTransaction
- public List<String> getTraitNames(String guid) throws MetadataException {
- LOG.info("Retrieving trait names for entity={}", guid);
- Vertex instanceVertex = getVertexForGUID(guid);
- return getTraitNames(instanceVertex);
- }
-
- public List<String> getTraitNames(Vertex entityVertex) {
- ArrayList<String> traits = new ArrayList<>();
- for (TitanProperty property : ((TitanVertex) entityVertex)
- .getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
- traits.add((String) property.getValue());
- }
-
- return traits;
- }
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitInstance trait instance that needs to be added to entity
- * @throws RepositoryException
- */
- @Override
- @GraphTransaction
- public void addTrait(String guid,
- ITypedStruct traitInstance) throws RepositoryException {
- Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
- final String traitName = traitInstance.getTypeName();
- LOG.info("Adding a new trait={} for entity={}", traitName, guid);
-
- try {
- Vertex instanceVertex = getVertexForGUID(guid);
-
- // add the trait instance as a new vertex
- final String typeName = getTypeName(instanceVertex);
- instanceToGraphMapper.mapTraitInstanceToVertex(
- traitInstance, getIdFromVertex(typeName, instanceVertex),
- typeName, instanceVertex, Collections.<Id, Vertex>emptyMap());
-
- // update the traits in entity once adding trait instance is successful
- ((TitanVertex) instanceVertex)
- .addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
-
- } catch (Exception e) {
- throw new RepositoryException(e);
- }
- }
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitNameToBeDeleted name of the trait
- * @throws RepositoryException
- */
- @Override
- @GraphTransaction
- public void deleteTrait(String guid, String traitNameToBeDeleted)
- throws RepositoryException {
- LOG.info("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
- try {
- Vertex instanceVertex = getVertexForGUID(guid);
-
- List<String> traitNames = getTraitNames(instanceVertex);
- if (!traitNames.contains(traitNameToBeDeleted)) {
- throw new RepositoryException("Could not find trait=" + traitNameToBeDeleted
- + " in the repository for entity: " + guid);
- }
-
- final String entityTypeName = getTypeName(instanceVertex);
- String relationshipLabel = getEdgeLabel(entityTypeName, traitNameToBeDeleted);
- Iterator<Edge> results = instanceVertex.getEdges(
- Direction.OUT, relationshipLabel).iterator();
- if (results.hasNext()) { // there should only be one edge for this label
- final Edge traitEdge = results.next();
- final Vertex traitVertex = traitEdge.getVertex(Direction.IN);
-
- // remove the edge to the trait instance from the repository
- titanGraph.removeEdge(traitEdge);
-
- if (traitVertex != null) { // remove the trait instance from the repository
- titanGraph.removeVertex(traitVertex);
-
- // update the traits in entity once trait removal is successful
- traitNames.remove(traitNameToBeDeleted);
- updateTraits(instanceVertex, traitNames);
- }
- }
- } catch (Exception e) {
- throw new RepositoryException(e);
- }
- }
-
- private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
- // remove the key
- instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
-
- // add it back again
- for (String traitName : traitNames) {
- ((TitanVertex) instanceVertex).addProperty(
- Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
- }
- }
-
- @Override
- @GraphTransaction
- public void updateEntity(String guid,
- String property, String value) throws RepositoryException {
- LOG.info("Adding property {} for entity guid {}", property, guid);
-
- try {
- Vertex instanceVertex = getVertexForGUID(guid);
-
- LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
- String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
- ClassType type = typeSystem.getDataType(ClassType.class, typeName);
- AttributeInfo attributeInfo = type.fieldMapping.fields.get(property);
- if (attributeInfo == null) {
- throw new MetadataException("Invalid property "
- + property + " for entity " + typeName);
- }
-
- DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
- ITypedReferenceableInstance instance = type.createInstance();
- if (attrTypeCategory == DataTypes.TypeCategory.PRIMITIVE) {
- instance.set(property, value);
- } else if (attrTypeCategory == DataTypes.TypeCategory.CLASS) {
- Id id = new Id(value, 0, attributeInfo.dataType().getName());
- instance.set(property, id);
- } else {
- throw new RepositoryException("Update of " + attrTypeCategory + " is not supported");
- }
-
- instanceToGraphMapper.mapAttributesToVertex(getIdFromVertex(typeName, instanceVertex),
- instance, instanceVertex, new HashMap<Id, Vertex>(),
- attributeInfo, attributeInfo.dataType());
- } catch (Exception e) {
- throw new RepositoryException(e);
- }
- }
-
- public Id getIdFromVertex(String dataTypeName, Vertex vertex) {
- return new Id(
- vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
- vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
- dataTypeName);
- }
-
- String getTypeName(Vertex instanceVertex) {
- return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
- }
-
-
- String getQualifiedName(ITypedInstance typedInstance,
- AttributeInfo attributeInfo) throws MetadataException {
- IDataType dataType = typeSystem.getDataType(
- IDataType.class, typedInstance.getTypeName());
- return getQualifiedName(dataType, attributeInfo.name);
- }
-
- String getQualifiedName(IDataType dataType,
- String attributeName) throws MetadataException {
- return dataType.getTypeCategory() == DataTypes.TypeCategory.STRUCT
- ? dataType.getName() + "." + attributeName
- // else class or trait
- : ((HierarchicalType) dataType).getQualifiedName(attributeName);
- }
-
- private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
-
- public final Map<Id, Id> idToNewIdMap;
- public final Map<Id, IReferenceableInstance> idToInstanceMap;
- public final Map<Id, Vertex> idToVertexMap;
-
- public EntityProcessor() {
- idToNewIdMap = new HashMap<>();
- idToInstanceMap = new HashMap<>();
- idToVertexMap = new HashMap<>();
- }
-
- @Override
- public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
- IReferenceableInstance ref = null;
- Id id = null;
-
- if (nd.attributeName == null) {
- ref = (IReferenceableInstance) nd.instance;
- id = ref.getId();
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (nd.value != null && (nd.value instanceof Id)) {
- id = (Id) nd.value;
- }
- }
-
- if (id != null) {
- if (id.isUnassigned()) {
- if (!idToNewIdMap.containsKey(id)) {
- idToNewIdMap.put(id, new Id(ID_SEQ.getAndIncrement(), 0, id.className));
- }
-
- if (ref != null) {
- if (idToInstanceMap.containsKey(id)) { // Oops
- throw new RepositoryException(String.format(
- "Unexpected internal error: Id %s processed again", id));
- }
-
- idToInstanceMap.put(id, ref);
- }
- }
- }
- }
-
- public void createVerticesForClassTypes(
- List<ITypedReferenceableInstance> newInstances) throws MetadataException {
- for (ITypedReferenceableInstance typedInstance : newInstances) {
- final Id id = typedInstance.getId();
- if (!idToVertexMap.containsKey(id)) {
- Vertex instanceVertex;
- if (id.isAssigned()) { // has a GUID
- instanceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
- } else {
- ClassType classType = typeSystem.getDataType(
- ClassType.class, typedInstance.getTypeName());
- instanceVertex = GraphHelper.createVertexWithIdentity(
- titanGraph, typedInstance,
- classType.getAllSuperTypeNames());
- }
-
- idToVertexMap.put(id, instanceVertex);
- }
- }
- }
- }
-
- private final class TypedInstanceToGraphMapper {
-
- private String mapTypedInstanceToGraph(IReferenceableInstance typedInstance)
- throws MetadataException {
-
- EntityProcessor entityProcessor = new EntityProcessor();
- try {
- LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
- new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
- } catch (MetadataException me) {
- throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
- }
-
- List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor);
- entityProcessor.createVerticesForClassTypes(newTypedInstances);
- String guid = addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances);
- addFullTextProperty(entityProcessor, newTypedInstances);
- return guid;
- }
-
- private void addFullTextProperty(EntityProcessor entityProcessor,
- List<ITypedReferenceableInstance> newTypedInstances)
- throws MetadataException {
-
- for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse
- Id id = typedInstance.getId();
- Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
- String fullText = getFullTextForVertex(instanceVertex, true);
- addProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
- }
- }
-
- private String getFullTextForVertex(Vertex instanceVertex,
- boolean followReferences) throws MetadataException {
- String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- ITypedReferenceableInstance typedReference =
- graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
- String fullText = getFullTextForInstance(typedReference, followReferences);
- StringBuilder fullTextBuilder = new StringBuilder(
- typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
-
- List<String> traits = typedReference.getTraits();
- for (String traitName : traits) {
- String traitText = getFullTextForInstance(
- (ITypedInstance) typedReference.getTrait(traitName), false);
- fullTextBuilder.append(FULL_TEXT_DELIMITER)
- .append(traitName)
- .append(FULL_TEXT_DELIMITER)
- .append(traitText);
- }
- return fullTextBuilder.toString();
- }
-
- private String getFullTextForAttribute(IDataType type, Object value,
- boolean followReferences) throws MetadataException {
- switch (type.getTypeCategory()) {
- case PRIMITIVE:
- return String.valueOf(value);
-
- case ENUM:
- return ((EnumValue)value).value;
-
- case ARRAY:
- StringBuilder fullText = new StringBuilder();
- IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
- List list = (List) value;
-
- for (Object element : list) {
- String elemFullText = getFullTextForAttribute(elemType, element, false);
- if (StringUtils.isNotEmpty(elemFullText))
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
- }
- return fullText.toString();
-
- case MAP:
- fullText = new StringBuilder();
- IDataType keyType = ((DataTypes.MapType) type).getKeyType();
- IDataType valueType = ((DataTypes.MapType) type).getValueType();
- Map map = (Map) value;
-
- for (Object entryObj : map.entrySet()) {
- Map.Entry entry = (Map.Entry) entryObj;
- String keyFullText = getFullTextForAttribute(keyType, entry.getKey(), false);
- if (StringUtils.isNotEmpty(keyFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
- }
- String valueFullText = getFullTextForAttribute(valueType, entry.getValue(), false);
- if (StringUtils.isNotEmpty(valueFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
- }
- }
- return fullText.toString();
-
- case CLASS:
- if (followReferences) {
- String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
- Vertex refVertex = getVertexForGUID(refGuid);
- return getFullTextForVertex(refVertex, false);
- }
- break;
-
- case STRUCT:
- if (followReferences) {
- return getFullTextForInstance((ITypedInstance) value, false);
- }
- break;
-
- default:
- throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
-
- }
- return null;
- }
-
- private String getFullTextForInstance(ITypedInstance typedInstance,
- boolean followReferences) throws MetadataException {
- StringBuilder fullText = new StringBuilder();
- for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
- Object attrValue = typedInstance.get(attributeInfo.name);
- if (attrValue == null) {
- continue;
- }
-
- String attrFullText = getFullTextForAttribute(
- attributeInfo.dataType(), attrValue, followReferences);
- if (StringUtils.isNotEmpty(attrFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name)
- .append(FULL_TEXT_DELIMITER).append(attrFullText);
- }
- }
- return fullText.toString();
- }
-
- /**
- * Step 2: Traverse oldIdToInstance map create newInstances :
- * List[ITypedReferenceableInstance]
- * - create a ITypedReferenceableInstance.
- * replace any old References ( ids or object references) with new Ids.
- */
- private List<ITypedReferenceableInstance> discoverInstances(EntityProcessor entityProcessor)
- throws RepositoryException {
- List<ITypedReferenceableInstance> newTypedInstances = new ArrayList<>();
- for (IReferenceableInstance transientInstance : entityProcessor.idToInstanceMap.values()) {
- LOG.debug("Discovered instance {}", transientInstance.getTypeName());
- try {
- ClassType cT = typeSystem.getDataType(
- ClassType.class, transientInstance.getTypeName());
- ITypedReferenceableInstance newInstance = cT.convert(
- transientInstance, Multiplicity.REQUIRED);
- newTypedInstances.add(newInstance);
-
- // Now replace old references with new Ids
- MapIds mapIds = new MapIds(entityProcessor.idToNewIdMap);
- new ObjectGraphWalker(typeSystem, mapIds, newTypedInstances).walk();
-
- } catch (MetadataException me) {
- throw new RepositoryException(
- String.format("Failed to create Instance(id = %s",
- transientInstance.getId()), me);
- }
- }
-
- return newTypedInstances;
- }
-
- private String addDiscoveredInstances(IReferenceableInstance entity,
- EntityProcessor entityProcessor,
- List<ITypedReferenceableInstance> newTypedInstances)
- throws MetadataException {
-
- String typedInstanceGUID = null;
- for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse over newInstances
- LOG.debug("Adding typed instance {}", typedInstance.getTypeName());
-
- Id id = typedInstance.getId();
- if (id == null) { // oops
- throw new RepositoryException("id cannot be null");
- }
-
- Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
-
- // add the attributes for the instance
- ClassType classType = typeSystem.getDataType(
- ClassType.class, typedInstance.getTypeName());
- final Map<String, AttributeInfo> fields = classType.fieldMapping().fields;
-
- mapInstanceToVertex(
- id, typedInstance, instanceVertex, fields, entityProcessor.idToVertexMap);
-
- for (String traitName : typedInstance.getTraits()) {
- LOG.debug("mapping trait {}", traitName);
- ((TitanVertex) instanceVertex)
- .addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
- ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
-
- // add the attributes for the trait instance
- mapTraitInstanceToVertex(traitInstance, typedInstance,
- instanceVertex, entityProcessor.idToVertexMap);
- }
-
- if (typedInstance.getId() == entity.getId()) { // save the guid for return
- typedInstanceGUID = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- }
- }
-
- return typedInstanceGUID;
- }
-
- private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
- Map<String, AttributeInfo> fields,
- Map<Id, Vertex> idToVertexMap) throws MetadataException {
- LOG.debug("Mapping instance {} of {} to vertex {}",
- typedInstance, typedInstance.getTypeName(), instanceVertex);
- for (AttributeInfo attributeInfo : fields.values()) {
- final IDataType dataType = attributeInfo.dataType();
- mapAttributesToVertex(id, typedInstance, instanceVertex,
- idToVertexMap, attributeInfo, dataType);
- }
- }
-
- private void mapAttributesToVertex(Id id, ITypedInstance typedInstance,
- Vertex instanceVertex,
- Map<Id, Vertex> idToVertexMap,
- AttributeInfo attributeInfo,
- IDataType dataType) throws MetadataException {
- Object attrValue = typedInstance.get(attributeInfo.name);
- LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
- final String propertyName = getQualifiedName(typedInstance, attributeInfo);
- String edgeLabel = getEdgeLabel(typedInstance, attributeInfo);
- if (attrValue == null) {
- return;
- }
-
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- mapPrimitiveToVertex(typedInstance, instanceVertex, attributeInfo);
- break;
-
- case ENUM:
- //handles both int and string for enum
- EnumValue enumValue = (EnumValue) dataType.convert(typedInstance.get(attributeInfo.name),
- Multiplicity.REQUIRED);
- addProperty(instanceVertex, propertyName, enumValue.value);
- break;
-
- case ARRAY:
- mapArrayCollectionToVertex(
- id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
- break;
-
- case MAP:
- mapMapCollectionToVertex(
- id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
- break;
-
- case STRUCT:
- Vertex structInstanceVertex = mapStructInstanceToVertex(id,
- (ITypedStruct) typedInstance.get(attributeInfo.name), attributeInfo, idToVertexMap);
- // add an edge to the newly created vertex from the parent
- GraphHelper.addEdge(
- titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
- break;
-
- case TRAIT:
- // do NOTHING - this is taken care of earlier
- break;
-
- case CLASS:
- Id referenceId = (Id) typedInstance.get(attributeInfo.name);
- mapClassReferenceAsEdge(
- instanceVertex, idToVertexMap, edgeLabel, referenceId);
- break;
-
- default:
- throw new IllegalArgumentException("Unknown type category: "
- + dataType.getTypeCategory());
- }
- }
-
- private void mapArrayCollectionToVertex(Id id, ITypedInstance typedInstance,
- Vertex instanceVertex,
- AttributeInfo attributeInfo,
- Map<Id, Vertex> idToVertexMap)
- throws MetadataException {
- LOG.debug("Mapping instance {} to vertex {} for name {}",
- typedInstance.getTypeName(), instanceVertex, attributeInfo.name);
- List list = (List) typedInstance.get(attributeInfo.name);
- if (list == null || list.isEmpty()) {
- return;
- }
-
- String propertyName = getQualifiedName(typedInstance, attributeInfo);
- IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
-
- List<String> values = new ArrayList<>(list.size());
- for (int index = 0; index < list.size(); index++) {
- String entryId = mapCollectionEntryToVertex(id, instanceVertex,
- attributeInfo, idToVertexMap, elementType, list.get(index), propertyName);
- values.add(entryId);
- }
-
- // for dereference on way out
- addProperty(instanceVertex, propertyName, values);
- }
-
- private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance,
- Vertex instanceVertex,
- AttributeInfo attributeInfo,
- Map<Id, Vertex> idToVertexMap)
- throws MetadataException {
- LOG.debug("Mapping instance {} to vertex {} for name {}",
- typedInstance.getTypeName(), instanceVertex, attributeInfo.name);
- @SuppressWarnings("unchecked")
- Map<Object, Object> collection =
- (Map<Object, Object>) typedInstance.get(attributeInfo.name);
- if (collection == null || collection.isEmpty()) {
- return;
- }
-
- String propertyName = getQualifiedName(typedInstance, attributeInfo);
- IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
- for (Map.Entry entry : collection.entrySet()) {
- String myPropertyName = propertyName + "." + entry.getKey().toString();
- String value = mapCollectionEntryToVertex(id, instanceVertex, attributeInfo,
- idToVertexMap, elementType, entry.getValue(), myPropertyName);
- addProperty(instanceVertex, myPropertyName, value);
- }
-
- // for dereference on way out
- addProperty(instanceVertex, propertyName, new ArrayList(collection.keySet()));
- }
-
- private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex,
- AttributeInfo attributeInfo,
- Map<Id, Vertex> idToVertexMap,
- IDataType elementType, Object value,
- String propertyName) throws MetadataException {
- final String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
- switch (elementType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- return value.toString();
-
- case ARRAY:
- case MAP:
- case TRAIT:
- // do nothing
- return null;
-
- case STRUCT:
- Vertex structInstanceVertex = mapStructInstanceToVertex(id,
- (ITypedStruct) value, attributeInfo, idToVertexMap);
- // add an edge to the newly created vertex from the parent
- Edge structElementEdge = GraphHelper.addEdge(
- titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
- return structElementEdge.getId().toString();
-
- case CLASS:
- Id referenceId = (Id) value;
- return mapClassReferenceAsEdge(
- instanceVertex, idToVertexMap, edgeLabel, referenceId);
-
- default:
- throw new IllegalArgumentException("Unknown type category: "
- + elementType.getTypeCategory());
- }
- }
-
- private String mapClassReferenceAsEdge(Vertex instanceVertex,
- Map<Id, Vertex> idToVertexMap,
- String propertyKey, Id id) throws MetadataException {
- if (id != null) {
- Vertex referenceVertex;
- if (id.isAssigned()) {
- referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
- } else {
- referenceVertex = idToVertexMap.get(id);
- }
-
- if (referenceVertex != null) {
- // add an edge to the class vertex from the instance
- Edge edge = GraphHelper.addEdge(
- titanGraph, instanceVertex, referenceVertex, propertyKey);
- return String.valueOf(edge.getId());
- }
- }
-
- return null;
- }
-
- private Vertex mapStructInstanceToVertex(Id id, ITypedStruct structInstance,
- AttributeInfo attributeInfo,
- Map<Id, Vertex> idToVertexMap)
- throws MetadataException {
- // add a new vertex for the struct or trait instance
- Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity(
- titanGraph, structInstance.getTypeName(), id,
- Collections.<String>emptySet()); // no super types for struct type
- LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name,
- structInstance);
-
- // map all the attributes to this newly created vertex
- mapInstanceToVertex(id, structInstance, structInstanceVertex,
- structInstance.fieldMapping().fields, idToVertexMap);
-
- return structInstanceVertex;
- }
-
- private void mapTraitInstanceToVertex(ITypedStruct traitInstance,
- ITypedReferenceableInstance typedInstance,
- Vertex parentInstanceVertex,
- Map<Id, Vertex> idToVertexMap)
- throws MetadataException {
- // add a new vertex for the struct or trait instance
- mapTraitInstanceToVertex(traitInstance, typedInstance.getId(),
- typedInstance.getTypeName(), parentInstanceVertex, idToVertexMap);
- }
-
- private void mapTraitInstanceToVertex(ITypedStruct traitInstance,
- Id typedInstanceId, String typedInstanceTypeName,
- Vertex parentInstanceVertex,
- Map<Id, Vertex> idToVertexMap)
- throws MetadataException {
- // add a new vertex for the struct or trait instance
- final String traitName = traitInstance.getTypeName();
- Vertex traitInstanceVertex = GraphHelper.createVertexWithoutIdentity(
- titanGraph, traitInstance.getTypeName(), typedInstanceId,
- typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
- LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
-
- // map all the attributes to this newly created vertex
- mapInstanceToVertex(typedInstanceId, traitInstance, traitInstanceVertex,
- traitInstance.fieldMapping().fields, idToVertexMap);
-
- // add an edge to the newly created vertex from the parent
- String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName);
- GraphHelper.addEdge(
- titanGraph, parentInstanceVertex, traitInstanceVertex, relationshipLabel);
- }
-
- private void mapPrimitiveToVertex(ITypedInstance typedInstance,
- Vertex instanceVertex,
- AttributeInfo attributeInfo) throws MetadataException {
- Object attrValue = typedInstance.get(attributeInfo.name);
- if (attrValue == null) {
- return; // add only if instance has this attribute
- }
-
- final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
- Object propertyValue = null;
- if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
- propertyValue = typedInstance.getString(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
- propertyValue = typedInstance.getShort(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
- propertyValue = typedInstance.getInt(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
- propertyValue = typedInstance.getBigInt(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
- propertyValue = typedInstance.getBoolean(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
- propertyValue = typedInstance.getByte(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
- propertyValue = typedInstance.getLong(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
- propertyValue = typedInstance.getFloat(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
- propertyValue = typedInstance.getDouble(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
- propertyValue = typedInstance.getDate(attributeInfo.name);
- }
- addProperty(instanceVertex, vertexPropertyName, propertyValue);
- }
- }
-
- private void addProperty(Vertex vertex, String propertyName, Object value) {
- LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
- vertex.setProperty(propertyName, value);
- }
-
- public final class GraphToTypedInstanceMapper {
-
- public ITypedReferenceableInstance mapGraphToTypedInstance(String guid,
- Vertex instanceVertex)
- throws MetadataException {
-
- LOG.debug("Mapping graph root vertex {} to typed instance for guid {}",
- instanceVertex, guid);
- String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
- List<String> traits = getTraitNames(instanceVertex);
-
- Id id = new Id(guid,
- instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
- LOG.debug("Created id {} for instance type {}", id, typeName);
-
- ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
- ITypedReferenceableInstance typedInstance = classType.createInstance(
- id, traits.toArray(new String[traits.size()]));
-
- mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
- mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
-
- return typedInstance;
- }
-
- public void mapVertexToInstanceTraits(Vertex instanceVertex,
- ITypedReferenceableInstance typedInstance,
- List<String> traits) throws MetadataException {
- for (String traitName : traits) {
- LOG.debug("mapping trait {} to instance", traitName);
- TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
- mapVertexToTraitInstance(
- instanceVertex, typedInstance, traitName, traitType);
- }
- }
-
- public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
- Map<String, AttributeInfo> fields)
- throws MetadataException {
-
- LOG.debug("Mapping vertex {} to instance {} for fields",
- instanceVertex, typedInstance.getTypeName(), fields);
- for (AttributeInfo attributeInfo : fields.values()) {
- mapVertexToAttribute(instanceVertex, typedInstance, attributeInfo);
- }
- }
-
- public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo) throws MetadataException {
- LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
- final IDataType dataType = attributeInfo.dataType();
- final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
-
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- mapVertexToPrimitive(instanceVertex, typedInstance, attributeInfo);
- break; // add only if vertex has this attribute
-
- case ENUM:
- if (instanceVertex.getProperty(vertexPropertyName) == null) {
- return;
- }
-
- typedInstance.set(attributeInfo.name, dataType.convert(instanceVertex.<String>getProperty
- (vertexPropertyName), Multiplicity.REQUIRED));
- break;
-
- case ARRAY:
- mapVertexToArrayInstance(
- instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
- break;
-
- case MAP:
- mapVertexToMapInstance(
- instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
- break;
-
- case STRUCT:
- mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
- break;
-
- case TRAIT:
- // do NOTHING - handled in class
- break;
-
- case CLASS:
- String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
- Object idOrInstance = mapClassReferenceToVertex(instanceVertex,
- attributeInfo, relationshipLabel, attributeInfo.dataType());
- typedInstance.set(attributeInfo.name, idOrInstance);
- break;
-
- default:
- break;
- }
- }
-
- public Object mapClassReferenceToVertex(Vertex instanceVertex,
- AttributeInfo attributeInfo,
- String relationshipLabel,
- IDataType dataType) throws MetadataException {
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- Iterator<Edge> results = instanceVertex.getEdges(
- Direction.OUT, relationshipLabel).iterator();
- if (results.hasNext()) {
- final Vertex referenceVertex = results.next().getVertex(Direction.IN);
- if (referenceVertex != null) {
- final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- LOG.debug("Found vertex {} for label {} with guid {}",
- referenceVertex, relationshipLabel, guid);
- if (attributeInfo.isComposite) {
- LOG.debug("Found composite, mapping vertex to instance");
- return mapGraphToTypedInstance(guid, referenceVertex);
- } else {
- Id referenceId = new Id(guid,
- referenceVertex.<Integer>getProperty( Constants.VERSION_PROPERTY_KEY),
- dataType.getName());
- LOG.debug("Found non-composite, adding id {} ", referenceId);
- return referenceId;
- }
- }
- }
-
- return null;
- }
-
- @SuppressWarnings("unchecked")
- public void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo,
- String propertyName) throws MetadataException {
- LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
- List list = instanceVertex.getProperty(propertyName);
- if (list == null || list.size() == 0) {
- return;
- }
- DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
- final IDataType elementType = arrayType.getElemType();
-
- ArrayList values = new ArrayList();
- for (Object listElement : list) {
- values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, listElement,
- propertyName));
- }
-
- typedInstance.set(attributeInfo.name, values);
- }
-
- public Object mapVertexToCollectionEntry(Vertex instanceVertex,
- AttributeInfo attributeInfo,
- IDataType elementType, Object value, String propertyName)
- throws MetadataException {
- String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
- switch (elementType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- return value;
-
- case ARRAY:
- case MAP:
- case TRAIT:
- // do nothing
- break;
-
- case STRUCT:
- return getStructInstanceFromVertex(instanceVertex,
- elementType, attributeInfo.name, edgeLabel, (String) value);
-
- case CLASS:
- return mapClassReferenceToVertex(
- instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
-
- default:
- break;
- }
-
- throw new IllegalArgumentException();
- }
-
- @SuppressWarnings("unchecked")
- private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo,
- final String propertyName) throws MetadataException {
- LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
- List<String> keys = instanceVertex.getProperty(propertyName);
- if (keys == null || keys.size() == 0) {
- return;
- }
- DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
- final IDataType valueType = mapType.getValueType();
-
- HashMap values = new HashMap();
- for (String key : keys) {
- String keyPropertyName = propertyName + "." + key;
- Object keyValue = instanceVertex.getProperty(keyPropertyName);
- values.put(key, mapVertexToCollectionEntry(instanceVertex, attributeInfo,
- valueType, keyValue, propertyName));
- }
-
- typedInstance.set(attributeInfo.name, values);
- }
-
- private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex,
- IDataType elemType,
- String attributeName, String relationshipLabel,
- String edgeId) throws MetadataException {
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- if (edgeId.equals(String.valueOf(edge.getId()))) {
- Vertex structInstanceVertex = edge.getVertex(Direction.IN);
- LOG.debug("mapping vertex {} to struct {}", structInstanceVertex,
- attributeName);
-
- if (structInstanceVertex != null) {
- LOG.debug("Found struct instance vertex {}, mapping to instance {} ",
- structInstanceVertex, elemType.getName());
- StructType structType = typeSystem
- .getDataType(StructType.class, elemType.getName());
- ITypedStruct structInstance = structType.createInstance();
-
- mapVertexToInstance(structInstanceVertex, structInstance,
- structType.fieldMapping().fields);
- return structInstance;
- }
-
- break;
- }
- }
-
- return null;
- }
-
- public Object mapClassReferenceToVertex(Vertex instanceVertex,
- AttributeInfo attributeInfo,
- String relationshipLabel,
- IDataType dataType,
- String edgeId) throws MetadataException {
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- if (edgeId.equals(String.valueOf(edge.getId()))) {
- final Vertex referenceVertex = edge.getVertex(Direction.IN);
- if (referenceVertex != null) {
- final String guid = referenceVertex
- .getProperty(Constants.GUID_PROPERTY_KEY);
- LOG.debug("Found vertex {} for label {} with guid {}",
- referenceVertex, relationshipLabel, guid);
- if (attributeInfo.isComposite) {
- LOG.debug("Found composite, mapping vertex to instance");
- return mapGraphToTypedInstance(guid, referenceVertex);
- } else {
- Id referenceId = new Id(guid,
- referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
- dataType.getName());
- LOG.debug("Found non-composite, adding id {} ", referenceId);
- return referenceId;
- }
- }
-
- break;
- }
- }
-
- return null;
- }
-
- private void mapVertexToStructInstance(Vertex instanceVertex,
- ITypedInstance typedInstance,
- AttributeInfo attributeInfo)
- throws MetadataException {
- LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
- StructType structType = typeSystem.getDataType(
- StructType.class, attributeInfo.dataType().getName());
- ITypedStruct structInstance = structType.createInstance();
- typedInstance.set(attributeInfo.name, structInstance);
-
- String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
- if (structInstanceVertex != null) {
- LOG.debug("Found struct instance vertex {}, mapping to instance {} ",
- structInstanceVertex, structInstance.getTypeName());
- mapVertexToInstance(structInstanceVertex, structInstance,
- structType.fieldMapping().fields);
- break;
- }
- }
- }
-
- private void mapVertexToTraitInstance(Vertex instanceVertex,
- ITypedReferenceableInstance typedInstance,
- String traitName,
- TraitType traitType) throws MetadataException {
- ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
-
- mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(),
- traitName, traitType, traitInstance);
- }
-
- public void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
- String traitName, TraitType traitType,
- ITypedStruct traitInstance) throws MetadataException {
- String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName);
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
- if (traitInstanceVertex != null) {
- LOG.debug("Found trait instance vertex {}, mapping to instance {} ",
- traitInstanceVertex, traitInstance.getTypeName());
- mapVertexToInstance(traitInstanceVertex, traitInstance,
- traitType.fieldMapping().fields);
- break;
- }
- }
- }
-
- private void mapVertexToPrimitive(Vertex instanceVertex,
- ITypedInstance typedInstance,
- AttributeInfo attributeInfo) throws MetadataException {
- LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
- final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
- if (instanceVertex.getProperty(vertexPropertyName) == null) {
- return;
- }
-
- if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
- typedInstance.setString(attributeInfo.name,
- instanceVertex.<String>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
- typedInstance.setShort(attributeInfo.name,
- instanceVertex.<Short>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
- typedInstance.setInt(attributeInfo.name,
- instanceVertex.<Integer>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
- typedInstance.setBigInt(attributeInfo.name,
- instanceVertex.<BigInteger>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
- typedInstance.setBoolean(attributeInfo.name,
- instanceVertex.<Boolean>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
- typedInstance.setByte(attributeInfo.name,
- instanceVertex.<Byte>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
- typedInstance.setLong(attributeInfo.name,
- instanceVertex.<Long>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
- typedInstance.setFloat(attributeInfo.name,
- instanceVertex.<Float>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
- typedInstance.setDouble(attributeInfo.name,
- instanceVertex.<Double>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- typedInstance.setBigDecimal(attributeInfo.name,
- instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
- typedInstance.setDate(attributeInfo.name,
- instanceVertex.<Date>getProperty(vertexPropertyName));
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
deleted file mode 100755
index 3b156ba..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedSearchIndexer.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.google.inject.Singleton;
-import com.thinkaurelius.titan.core.Cardinality;
-import com.thinkaurelius.titan.core.EdgeLabel;
-import com.thinkaurelius.titan.core.Order;
-import com.thinkaurelius.titan.core.PropertyKey;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.schema.Mapping;
-import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
-import com.thinkaurelius.titan.core.schema.TitanManagement;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.discovery.SearchIndexer;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-
-/**
- * Adds index for properties of a given type when its added before any instances are added.
- */
-@Singleton
-public class GraphBackedSearchIndexer implements SearchIndexer {
-
- private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
-
- private final TitanGraph titanGraph;
-
- @Inject
- public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
- throws RepositoryException {
-
- this.titanGraph = graphProvider.get();
-
- initialize();
- }
-
- /**
- * Initializes the indices for the graph - create indices for Global Vertex Keys
- */
- private void initialize() {
- TitanManagement management = titanGraph.getManagementSystem();
- if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
- LOG.info("Global indexes already exist for graph");
- return;
- }
-
- LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
- management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
- .buildMixedIndex(Constants.BACKING_INDEX);
- management.buildIndex(Constants.EDGE_INDEX, Edge.class)
- .buildMixedIndex(Constants.BACKING_INDEX);
- management.commit();
-
- // create a composite index for guid as its unique
- createCompositeIndex(Constants.GUID_INDEX,
- Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE);
-
- // create a composite and mixed index for type since it can be combined with other keys
- createCompositeAndMixedIndex(Constants.ENTITY_TYPE_INDEX,
- Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE);
-
- // create a composite and mixed index for type since it can be combined with other keys
- createCompositeAndMixedIndex(Constants.SUPER_TYPES_INDEX,
- Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET);
-
- // create a composite and mixed index for traitNames since it can be combined with other
- // keys. Traits must be a set and not a list.
- createCompositeAndMixedIndex(Constants.TRAIT_NAMES_INDEX,
- Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET);
-
- // Index for full text search
- createFullTextIndex();
-
- //Indexes for graph backed type system store
- createTypeStoreIndexes();
-
- LOG.info("Index creation for global keys complete.");
- }
-
- private void createFullTextIndex() {
- TitanManagement management = titanGraph.getManagementSystem();
- PropertyKey fullText =
- management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();
-
- management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
- .addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
- .buildMixedIndex(Constants.BACKING_INDEX);
- management.commit();
- LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
- }
-
- private void createTypeStoreIndexes() {
- //Create unique index on typeName
- createCompositeIndex(Constants.TYPENAME_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY, String.class,
- true, Cardinality.SINGLE);
-
- //create index on vertex type
- createCompositeIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class,
- false, Cardinality.SINGLE);
-
- }
-
- /**
- * This is upon adding a new type to Store.
- *
- * @param typeName type name
- * @param dataType data type
- * @throws org.apache.hadoop.metadata.MetadataException
- */
- @Override
- public void onAdd(String typeName, IDataType dataType) throws MetadataException {
- LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);
-
- try {
- addIndexForType(dataType);
- LOG.info("Index creation for type {} complete", typeName);
-
- } catch (Throwable throwable) {
- // gets handle to currently open transaction
- titanGraph.getManagementSystem().rollback();
- LOG.error("Error creating index for type {}", dataType, throwable);
- }
- }
-
- private void addIndexForType(IDataType dataType) {
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- case ARRAY:
- case MAP:
- // do nothing since these are only attributes
- // and not types like structs, traits or classes
- break;
-
- case STRUCT:
- StructType structType = (StructType) dataType;
- createIndexForFields(structType, structType.fieldMapping().fields);
- break;
-
- case TRAIT:
- TraitType traitType = (TraitType) dataType;
- createIndexForFields(traitType, traitType.fieldMapping().fields);
- break;
-
- case CLASS:
- ClassType classType = (ClassType) dataType;
- createIndexForFields(classType, classType.fieldMapping().fields);
- break;
-
- default:
- throw new IllegalArgumentException("bad data type" + dataType);
- }
- }
-
- private void createIndexForFields(IDataType dataType, Map<String, AttributeInfo> fields) {
- for (AttributeInfo field : fields.values()) {
- if (field.isIndexable) {
- createIndexForAttribute(dataType.getName(), field);
- }
- }
- }
-
- private void createIndexForAttribute(String typeName, AttributeInfo field) {
- final String propertyName = typeName + "." + field.name;
- switch (field.dataType().getTypeCategory()) {
- case PRIMITIVE:
- createVertexMixedIndex(propertyName, getPrimitiveClass(field.dataType()));
- break;
-
- case ENUM:
- createVertexMixedIndex(propertyName, String.class);
- break;
-
- case ARRAY:
- case MAP:
- // todo - how do we overcome this limitation?
- // IGNORE: Can only index single-valued property keys on vertices in Mixed Index
- break;
-
- case STRUCT:
- StructType structType = (StructType) field.dataType();
- createIndexForFields(structType, structType.fieldMapping().fields);
- break;
-
- case TRAIT:
- // do nothing since this is NOT contained in other types
- break;
-
- case CLASS:
- // this is only A reference, index the attribute for edge
- // Commenting this out since we do not need an index for edge here
- //createEdgeMixedIndex(propertyName);
- break;
-
- default:
- throw new IllegalArgumentException("bad data type" + field.dataType().getName());
- }
- }
-
- private Class getPrimitiveClass(IDataType dataType) {
- if (dataType == DataTypes.STRING_TYPE) {
- return String.class;
- } else if (dataType == DataTypes.SHORT_TYPE) {
- return Short.class;
- } else if (dataType == DataTypes.INT_TYPE) {
- return Integer.class;
- } else if (dataType == DataTypes.BIGINTEGER_TYPE) {
- return BigInteger.class;
- } else if (dataType == DataTypes.BOOLEAN_TYPE) {
- return Boolean.class;
- } else if (dataType == DataTypes.BYTE_TYPE) {
- return Byte.class;
- } else if (dataType == DataTypes.LONG_TYPE) {
- return Long.class;
- } else if (dataType == DataTypes.FLOAT_TYPE) {
- return Float.class;
- } else if (dataType == DataTypes.DOUBLE_TYPE) {
- return Double.class;
- } else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
- return BigDecimal.class;
- } else if (dataType == DataTypes.DATE_TYPE) {
- return Date.class;
- }
-
-
- throw new IllegalArgumentException("unknown data type " + dataType);
- }
-
-/*
- private Cardinality getCardinality(Multiplicity multiplicity) {
- if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
- return Cardinality.SINGLE;
- } else if (multiplicity == Multiplicity.COLLECTION) {
- return Cardinality.LIST;
- } else if (multiplicity == Multiplicity.SET) {
- return Cardinality.SET;
- }
-
- // todo - default to LIST as this is the most forgiving
- return Cardinality.LIST;
- }
-*/
-
- private void createCompositeAndMixedIndex(String indexName,
- String propertyName, Class propertyClass,
- boolean isUnique, Cardinality cardinality) {
- createCompositeIndex(indexName, propertyName, propertyClass, isUnique, cardinality);
- createVertexMixedIndex(propertyName, propertyClass);
- }
-
- private PropertyKey createCompositeIndex(String indexName,
- String propertyName, Class propertyClass,
- boolean isUnique, Cardinality cardinality) {
- TitanManagement management = titanGraph.getManagementSystem();
- PropertyKey propertyKey = management.getPropertyKey(propertyName);
- if (propertyKey == null) {
- propertyKey = management
- .makePropertyKey(propertyName)
- .dataType(propertyClass)
- .cardinality(cardinality)
- .make();
-
- TitanManagement.IndexBuilder indexBuilder = management
- .buildIndex(indexName, Vertex.class)
- .addKey(propertyKey);
-
- if (isUnique) {
- indexBuilder = indexBuilder.unique();
- }
-
- indexBuilder.buildCompositeIndex();
- management.commit();
-
- LOG.info("Created index for property {} in composite index {}", propertyName, indexName);
- }
-
- return propertyKey;
- }
-
- private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
- TitanManagement management = titanGraph.getManagementSystem();
- PropertyKey propertyKey = management.getPropertyKey(propertyName);
- if (propertyKey == null) {
- // ignored cardinality as Can only index single-valued property keys on vertices
- propertyKey = management
- .makePropertyKey(propertyName)
- .dataType(propertyClass)
- .make();
-
- if (propertyClass == Boolean.class) {
- //Use standard index as backing index only supports string, int and geo types
- management.buildIndex(propertyName, Vertex.class).addKey(propertyKey).buildCompositeIndex();
- management.commit();
- } else {
- //Use backing index
- TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
- management.addIndexKey(vertexIndex, propertyKey);
- management.commit();
- }
- LOG.info("Created mixed vertex index for property {}", propertyName);
- }
-
- return propertyKey;
- }
-
- /* Commenting this out since we do not need an index for edge label here
- private void createEdgeMixedIndex(String propertyName) {
- TitanManagement management = titanGraph.getManagementSystem();
- EdgeLabel edgeLabel = management.getEdgeLabel(propertyName);
- if (edgeLabel == null) {
- edgeLabel = management.makeEdgeLabel(propertyName).make();
- management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
- management.commit();
- LOG.info("Created index for edge label {}", propertyName);
- }
- } */
-}
[29/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
new file mode 100755
index 0000000..b475407
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import com.thinkaurelius.titan.core.TitanVertex
+import com.tinkerpop.blueprints.Direction
+import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
+import org.apache.atlas.query.TypeUtils.FieldInfo
+import org.apache.atlas.typesystem.persistence.Id
+import org.apache.atlas.typesystem.types.DataTypes._
+import org.apache.atlas.typesystem.types._
+import org.apache.atlas.typesystem.{ITypedInstance, ITypedReferenceableInstance}
+
+import scala.collection.JavaConversions._
+
+/**
+ * Represents the Bridge between the QueryProcessor and the Graph Persistence scheme used.
+ * Some of the behaviors captured are:
+ * - how is type and id information stored in the Vertex that represents an [[ITypedReferenceableInstance]]
+ * - how are edges representing trait and attribute relationships labelled.
+ * - how are attribute names mapped to Property Keys in Vertices.
+ *
+ * This is a work in progress.
+ */
+trait GraphPersistenceStrategies {
+ /**
+ * Name of attribute used to store typeName in vertex
+ */
+ def typeAttributeName: String
+
+ /**
+ * Name of attribute used to store super type names in vertex.
+ */
+ def superTypeAttributeName: String
+
+ /**
+ * Name of attribute used to store guid in vertex
+ */
+ def idAttributeName : String
+
+ /**
+ * Given a dataType and a reference attribute, how is edge labeled
+ */
+ def edgeLabel(iDataType: IDataType[_], aInfo: AttributeInfo): String
+
+ def traitLabel(cls: IDataType[_], traitName: String): String
+
+ def instanceToTraitEdgeDirection : String = "out"
+ def traitToInstanceEdgeDirection = instanceToTraitEdgeDirection match {
+ case "out" => "in"
+ case "in" => "out"
+ case x => x
+ }
+
+ /**
+ * The propertyKey used to store the attribute in a Graph Vertex.
+ * @param dataType
+ * @param aInfo
+ * @return
+ */
+ def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo): String
+
+ /**
+ * from a vertex for an [[ITypedReferenceableInstance]] get the traits that it has.
+ * @param v
+ * @return
+ */
+ def traitNames(v: TitanVertex): java.util.List[String]
+
+ def edgeLabel(fInfo: FieldInfo): String = fInfo match {
+ case FieldInfo(dataType, aInfo, null, null) => edgeLabel(dataType, aInfo)
+ case FieldInfo(dataType, aInfo, reverseDataType, null) => edgeLabel(reverseDataType, aInfo)
+ case FieldInfo(dataType, null, null, traitName) => traitLabel(dataType, traitName)
+ }
+
+ def fieldPrefixInSelect: String
+
+ /**
+ * extract the Id from a Vertex.
+ * @param dataTypeNm the dataType of the instance that the given vertex represents
+ * @param v
+ * @return
+ */
+ def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id
+
+ def constructInstance[U](dataType: IDataType[U], v: java.lang.Object): U
+
+ def gremlinCompOp(op: ComparisonExpression) = op.symbol match {
+ case "=" => "T.eq"
+ case "!=" => "T.neq"
+ case ">" => "T.gt"
+ case ">=" => "T.gte"
+ case "<" => "T.lt"
+ case "<=" => "T.lte"
+ case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin")
+ }
+
+ def loopObjectExpression(dataType: IDataType[_]) = {
+ _typeTestExpression(dataType.getName, "it.object")
+ }
+
+ def addGraphVertexPrefix(preStatements : Traversable[String]) = !collectTypeInstancesIntoVar
+
+ /**
+ * Controls behavior of how instances of a Type are discovered.
+ * - query is generated in a way that indexes are exercised using a local set variable across multiple lookups
+ * - query is generated using an 'or' expression.
+ *
+ * '''This is a very bad idea: controlling query execution behavior via query generation.''' But our current
+ * knowledge of seems to indicate we have no choice. See
+ * [[https://groups.google.com/forum/#!topic/gremlin-users/n1oV86yr4yU discussion in Gremlin group]].
+ * Also this seems a fragile solution, dependend on the memory requirements of the Set variable.
+ * For now enabling via the '''collectTypeInstancesIntoVar''' behavior setting. Reverting back would require
+ * setting this to false.
+ *
+ * Long term have to get to the bottom of Gremlin:
+ * - there doesn't seem to be way to see the physical query plan. Maybe we should directly interface with Titan.
+ * - At least from querying perspective a columnar db maybe a better route. Daniel Abadi did some good work
+ * on showing how to use a columnar store as a Graph Db.
+ *
+ *
+ * @return
+ */
+ def collectTypeInstancesIntoVar = true
+
+ def typeTestExpression(typeName : String, intSeq : IntSequence) : Seq[String] = {
+ if (collectTypeInstancesIntoVar)
+ typeTestExpressionMultiStep(typeName, intSeq)
+ else
+ typeTestExpressionUsingFilter(typeName)
+ }
+
+ private def typeTestExpressionUsingFilter(typeName : String) : Seq[String] = {
+ Seq(s"""filter${_typeTestExpression(typeName, "it")}""")
+ }
+
+ private def _typeTestExpression(typeName: String, itRef: String): String = {
+ s"""{(${itRef}.'${typeAttributeName}' == '${typeName}') |
+ |(${itRef}.'${superTypeAttributeName}' ?
+ |${itRef}.'${superTypeAttributeName}'.contains('${typeName}') : false)}""".
+ stripMargin.replace(System.getProperty("line.separator"), "")
+ }
+
+ private def typeTestExpressionMultiStep(typeName : String, intSeq : IntSequence) : Seq[String] = {
+
+ val varName = s"_var_${intSeq.next}"
+ Seq(
+ newSetVar(varName),
+ fillVarWithTypeInstances(typeName, varName),
+ fillVarWithSubTypeInstances(typeName, varName),
+ s"$varName._()"
+ )
+ }
+
+ private def newSetVar(varName : String) = s"$varName = [] as Set"
+
+ private def fillVarWithTypeInstances(typeName : String, fillVar : String) = {
+ s"""g.V().has("${typeAttributeName}", "${typeName}").fill($fillVar)"""
+ }
+
+ private def fillVarWithSubTypeInstances(typeName : String, fillVar : String) = {
+ s"""g.V().has("${superTypeAttributeName}", "${typeName}").fill($fillVar)"""
+ }
+}
+
+object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
+ val typeAttributeName = "typeName"
+ val superTypeAttributeName = "superTypeNames"
+ val idAttributeName = "guid"
+
+ def edgeLabel(dataType: IDataType[_], aInfo: AttributeInfo) = s"${dataType.getName}.${aInfo.name}"
+
+ val fieldPrefixInSelect = "it"
+
+ def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName"
+
+ def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = aInfo.name
+
+ def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id =
+ new Id(v.getId.toString, 0, dataTypeNm)
+
+ def traitNames(v: TitanVertex): java.util.List[String] = {
+ val s = v.getProperty[String]("traitNames")
+ if (s != null) {
+ Seq[String](s.split(","): _*)
+ } else {
+ Seq()
+ }
+ }
+
+ def constructInstance[U](dataType: IDataType[U], v: AnyRef): U = {
+ dataType.getTypeCategory match {
+ case DataTypes.TypeCategory.PRIMITIVE => dataType.convert(v, Multiplicity.OPTIONAL)
+ case DataTypes.TypeCategory.STRUCT
+ if dataType.getName == TypeSystem.getInstance().getIdType.getName => {
+ val sType = dataType.asInstanceOf[StructType]
+ val sInstance = sType.createInstance()
+ val tV = v.asInstanceOf[TitanVertex]
+ sInstance.set(TypeSystem.getInstance().getIdType.typeNameAttrName,
+ tV.getProperty[java.lang.String](typeAttributeName))
+ sInstance.set(TypeSystem.getInstance().getIdType.idAttrName,
+ tV.getProperty[java.lang.String](idAttributeName))
+ dataType.convert(sInstance, Multiplicity.OPTIONAL)
+ }
+ case DataTypes.TypeCategory.STRUCT => {
+ val sType = dataType.asInstanceOf[StructType]
+ val sInstance = sType.createInstance()
+ loadStructInstance(sType, sInstance, v.asInstanceOf[TitanVertex])
+ dataType.convert(sInstance, Multiplicity.OPTIONAL)
+ }
+ case DataTypes.TypeCategory.TRAIT => {
+ val tType = dataType.asInstanceOf[TraitType]
+ val tInstance = tType.createInstance()
+ /*
+ * this is not right, we should load the Instance associated with this trait.
+ * for now just loading the trait struct.
+ */
+ loadStructInstance(tType, tInstance, v.asInstanceOf[TitanVertex])
+ dataType.convert(tInstance, Multiplicity.OPTIONAL)
+ }
+ case DataTypes.TypeCategory.CLASS => {
+ val cType = dataType.asInstanceOf[ClassType]
+ val cInstance = constructClassInstance(dataType.asInstanceOf[ClassType], v.asInstanceOf[TitanVertex])
+ dataType.convert(cInstance, Multiplicity.OPTIONAL)
+ }
+ case DataTypes.TypeCategory.ENUM => dataType.convert(v, Multiplicity.OPTIONAL)
+ case x => throw new UnsupportedOperationException(s"load for ${dataType} not supported")
+ }
+ }
+
+ def loadStructInstance(dataType: IConstructableType[_, _ <: ITypedInstance],
+ typInstance: ITypedInstance, v: TitanVertex): Unit = {
+ import scala.collection.JavaConversions._
+ dataType.fieldMapping().fields.foreach { t =>
+ val fName = t._1
+ val aInfo = t._2
+ loadAttribute(dataType, aInfo, typInstance, v)
+ }
+ }
+
+ def constructClassInstance(dataType: ClassType, v: TitanVertex): ITypedReferenceableInstance = {
+ val id = getIdFromVertex(dataType.name, v)
+ val tNms = traitNames(v)
+ val cInstance = dataType.createInstance(id, tNms: _*)
+ // load traits
+ tNms.foreach { tNm =>
+ val tLabel = traitLabel(dataType, tNm)
+ val edges = v.getEdges(Direction.OUT, tLabel)
+ val tVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex]
+ val tType = TypeSystem.getInstance().getDataType[TraitType](classOf[TraitType], tNm)
+ val tInstance = cInstance.getTrait(tNm).asInstanceOf[ITypedInstance]
+ loadStructInstance(tType, tInstance, tVertex)
+ }
+ loadStructInstance(dataType, cInstance, v)
+ cInstance
+ }
+
+ def loadAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex): Unit = {
+ aInfo.dataType.getTypeCategory match {
+ case DataTypes.TypeCategory.PRIMITIVE => loadPrimitiveAttribute(dataType, aInfo, i, v)
+ case DataTypes.TypeCategory.ENUM => loadEnumAttribute(dataType, aInfo, i, v)
+ case DataTypes.TypeCategory.ARRAY =>
+ throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
+ case DataTypes.TypeCategory.MAP =>
+ throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
+ case DataTypes.TypeCategory.STRUCT => loadStructAttribute(dataType, aInfo, i, v)
+ case DataTypes.TypeCategory.TRAIT =>
+ throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
+ case DataTypes.TypeCategory.CLASS => loadStructAttribute(dataType, aInfo, i, v)
+ }
+ }
+
+ private def loadEnumAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex)
+ : Unit = {
+ val fName = fieldNameInVertex(dataType, aInfo)
+ i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName))
+ }
+
+ private def loadPrimitiveAttribute(dataType: IDataType[_], aInfo: AttributeInfo,
+ i: ITypedInstance, v: TitanVertex): Unit = {
+ val fName = fieldNameInVertex(dataType, aInfo)
+ aInfo.dataType() match {
+ case x: BooleanType => i.setBoolean(aInfo.name, v.getProperty[java.lang.Boolean](fName))
+ case x: ByteType => i.setByte(aInfo.name, v.getProperty[java.lang.Byte](fName))
+ case x: ShortType => i.setShort(aInfo.name, v.getProperty[java.lang.Short](fName))
+ case x: IntType => i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName))
+ case x: LongType => i.setLong(aInfo.name, v.getProperty[java.lang.Long](fName))
+ case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName))
+ case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName))
+ case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName))
+ case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported")
+ }
+ }
+
+ private def loadStructAttribute(dataType: IDataType[_], aInfo: AttributeInfo,
+ i: ITypedInstance, v: TitanVertex): Unit = {
+ val eLabel = edgeLabel(FieldInfo(dataType, aInfo, null))
+ val edges = v.getEdges(Direction.OUT, eLabel)
+ val sVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex]
+ if (aInfo.dataType().getTypeCategory == DataTypes.TypeCategory.STRUCT) {
+ val sType = aInfo.dataType().asInstanceOf[StructType]
+ val sInstance = sType.createInstance()
+ loadStructInstance(sType, sInstance, sVertex)
+ i.set(aInfo.name, sInstance)
+ } else {
+ val cInstance = constructClassInstance(aInfo.dataType().asInstanceOf[ClassType], sVertex)
+ i.set(aInfo.name, cInstance)
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala
new file mode 100755
index 0000000..edb190d
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/GremlinEvaluator.scala
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
+
+import com.thinkaurelius.titan.core.TitanGraph
+import com.tinkerpop.pipes.util.structures.Row
+import org.apache.atlas.query.TypeUtils.ResultWithPathStruct
+import org.apache.atlas.typesystem.json._
+import org.apache.atlas.typesystem.types._
+import org.json4s._
+import org.json4s.native.Serialization._
+
+import scala.language.existentials
+
+case class GremlinQueryResult(query: String,
+ resultDataType: IDataType[_],
+ rows: List[_]) {
+ def toJson = JsonHelper.toJson(this)
+}
+
+class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: TitanGraph) {
+
+ val manager: ScriptEngineManager = new ScriptEngineManager
+ val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
+ val bindings: Bindings = engine.createBindings
+ bindings.put("g", g)
+
+ /**
+ *
+ * @param gResultObj is the object returned from gremlin. This must be a List
+ * @param qryResultObj is the object constructed for the output w/o the Path.
+ * @return a ResultWithPathStruct
+ */
+ def addPathStruct(gResultObj : AnyRef, qryResultObj : Any) : Any = {
+ if ( !qry.isPathExpresion) {
+ qryResultObj
+ } else {
+ import scala.collection.JavaConversions._
+ import scala.collection.JavaConverters._
+ val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init
+
+ val oPaths = iPaths.map { p =>
+ persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, p)
+ }.toList.asJava
+ val sType = qry.expr.dataType.asInstanceOf[StructType]
+ val sInstance = sType.createInstance()
+ sInstance.set(ResultWithPathStruct.pathAttrName, oPaths)
+ sInstance.set(ResultWithPathStruct.resultAttrName, qryResultObj)
+ sInstance
+ }
+ }
+
+ def instanceObject(v : AnyRef) : AnyRef = {
+ if ( qry.isPathExpresion ) {
+ import scala.collection.JavaConversions._
+ v.asInstanceOf[java.util.List[AnyRef]].last
+ } else {
+ v
+ }
+ }
+
+ def evaluate(): GremlinQueryResult = {
+ import scala.collection.JavaConversions._
+ val rType = qry.expr.dataType
+ val oType = if (qry.isPathExpresion) qry.expr.children(0).dataType else rType
+ val rawRes = engine.eval(qry.queryStr, bindings)
+
+ if (!qry.hasSelectList) {
+ val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { v =>
+ val iV = instanceObject(v)
+ val o = persistenceStrategy.constructInstance(oType, iV)
+ addPathStruct(v, o)
+ }
+ GremlinQueryResult(qry.expr.toString, rType, rows.toList)
+ } else {
+ val sType = oType.asInstanceOf[StructType]
+ val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { r =>
+ val rV = instanceObject(r).asInstanceOf[Row[java.util.List[AnyRef]]]
+ val sInstance = sType.createInstance()
+ val selExpr =
+ (if (qry.isPathExpresion) qry.expr.children(0) else qry.expr).
+ asInstanceOf[Expressions.SelectExpression]
+ selExpr.selectListWithAlias.foreach { aE =>
+ val cName = aE.alias
+ val (src, idx) = qry.resultMaping(cName)
+ val v = rV.getColumn(src).get(idx)
+ sInstance.set(cName, persistenceStrategy.constructInstance(aE.dataType, v))
+ }
+ addPathStruct(r, sInstance)
+ }
+ GremlinQueryResult(qry.expr.toString, rType, rows.toList)
+ }
+
+ }
+}
+
+object JsonHelper {
+
+ class GremlinQueryResultSerializer()
+ extends Serializer[GremlinQueryResult] {
+ def deserialize(implicit format: Formats) = {
+ throw new UnsupportedOperationException("Deserialization of GremlinQueryResult not supported")
+ }
+
+ def serialize(implicit f: Formats) = {
+ case GremlinQueryResult(query, rT, rows) =>
+ JObject(JField("query", JString(query)),
+ JField("dataType", TypesSerialization.toJsonValue(rT)(f)),
+ JField("rows", Extraction.decompose(rows)(f))
+ )
+ }
+ }
+
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer +
+ new GremlinQueryResultSerializer
+
+ def toJson(r: GremlinQueryResult): String = {
+ writePretty(r)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
new file mode 100755
index 0000000..914ed7a
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import org.apache.atlas.query.Expressions._
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory
+
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
+
+trait IntSequence {
+ def next: Int
+}
+
+case class GremlinQuery(expr: Expression, queryStr: String, resultMaping: Map[String, (String, Int)]) {
+
+ def hasSelectList = resultMaping != null
+
+ def isPathExpresion = expr.isInstanceOf[PathExpression]
+}
+
+trait SelectExpressionHandling {
+
+ /**
+ * To aide in gremlinQuery generation add an alias to the input of SelectExpressions
+ */
+ class AddAliasToSelectInput extends PartialFunction[Expression, Expression] {
+
+ private var idx = 0
+
+ def isDefinedAt(e: Expression) = true
+
+ class DecorateFieldWithAlias(aliasE: AliasExpression)
+ extends PartialFunction[Expression, Expression] {
+ def isDefinedAt(e: Expression) = true
+
+ def apply(e: Expression) = e match {
+ case fe@FieldExpression(fieldName, fInfo, None) =>
+ FieldExpression(fieldName, fInfo, Some(BackReference(aliasE.alias, aliasE.child, None)))
+ case _ => e
+ }
+ }
+
+ def apply(e: Expression) = e match {
+ case SelectExpression(aliasE@AliasExpression(_, _), selList) => {
+ idx = idx + 1
+ SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE))))
+ }
+ case SelectExpression(child, selList) => {
+ idx = idx + 1
+ val aliasE = AliasExpression(child, s"_src$idx")
+ SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE))))
+ }
+ case _ => e
+ }
+ }
+
+ def getSelectExpressionSrc(e: Expression): List[String] = {
+ val l = ArrayBuffer[String]()
+ e.traverseUp {
+ case BackReference(alias, _, _) => l += alias
+ }
+ l.toSet.toList
+ }
+
+ def validateSelectExprHaveOneSrc: PartialFunction[Expression, Unit] = {
+ case SelectExpression(_, selList) => {
+ selList.foreach { se =>
+ val srcs = getSelectExpressionSrc(se)
+ if (srcs.size > 1) {
+ throw new GremlinTranslationException(se, "Only one src allowed in a Select Expression")
+ }
+ }
+ }
+ }
+
+ def groupSelectExpressionsBySrc(sel: SelectExpression): mutable.LinkedHashMap[String, List[Expression]] = {
+ val m = mutable.LinkedHashMap[String, List[Expression]]()
+ sel.selectListWithAlias.foreach { se =>
+ val l = getSelectExpressionSrc(se.child)
+ if (!m.contains(l(0))) {
+ m(l(0)) = List()
+ }
+ m(l(0)) = m(l(0)) :+ se.child
+ }
+ m
+ }
+
+ /**
+ * For each Output Column in the SelectExpression compute the ArrayList(Src) this maps to and the position within
+ * this list.
+ * @param sel
+ * @return
+ */
+ def buildResultMapping(sel: SelectExpression): Map[String, (String, Int)] = {
+ val srcToExprs = groupSelectExpressionsBySrc(sel)
+ val m = new mutable.HashMap[String, (String, Int)]
+ sel.selectListWithAlias.foreach { se =>
+ val src = getSelectExpressionSrc(se.child)(0)
+ val srcExprs = srcToExprs(src)
+ var idx = srcExprs.indexOf(se.child)
+ m(se.alias) = (src, idx)
+ }
+ m.toMap
+ }
+
+}
+
+class GremlinTranslationException(expr: Expression, reason: String) extends
+ExpressionException(expr, s"Unsupported Gremlin translation: $reason")
+
+class GremlinTranslator(expr: Expression,
+ gPersistenceBehavior: GraphPersistenceStrategies)
+ extends SelectExpressionHandling {
+
+ val preStatements = ArrayBuffer[String]()
+ val postStatements = ArrayBuffer[String]()
+
+ val wrapAndRule: PartialFunction[Expression, Expression] = {
+ case f: FilterExpression if !f.condExpr.isInstanceOf[LogicalExpression] =>
+ FilterExpression(f.child, new LogicalExpression("and", List(f.condExpr)))
+ }
+
+ val validateComparisonForm: PartialFunction[Expression, Unit] = {
+ case c@ComparisonExpression(_, left, right) =>
+ if (!left.isInstanceOf[FieldExpression]) {
+ throw new GremlinTranslationException(c, s"lhs of comparison is not a field")
+ }
+ if (!right.isInstanceOf[Literal[_]]) {
+ throw new GremlinTranslationException(c,
+ s"rhs of comparison is not a literal")
+ }
+ ()
+ }
+
+ val counter = new IntSequence {
+ var i: Int = -1;
+
+ def next: Int = {
+ i += 1; i
+ }
+ }
+
+ def addAliasToLoopInput(c: IntSequence = counter): PartialFunction[Expression, Expression] = {
+ case l@LoopExpression(aliasE@AliasExpression(_, _), _, _) => l
+ case l@LoopExpression(inputExpr, loopExpr, t) => {
+ val aliasE = AliasExpression(inputExpr, s"_loop${c.next}")
+ LoopExpression(aliasE, loopExpr, t)
+ }
+ }
+
+ def instanceClauseToTop(topE : Expression) : PartialFunction[Expression, Expression] = {
+ case le : LogicalExpression if (le fastEquals topE) => {
+ le.instance()
+ }
+ case ce : ComparisonExpression if (ce fastEquals topE) => {
+ ce.instance()
+ }
+ case he : hasFieldUnaryExpression if (he fastEquals topE) => {
+ he.instance()
+ }
+ }
+
+ def traitClauseWithInstanceForTop(topE : Expression) : PartialFunction[Expression, Expression] = {
+ case te : TraitExpression if (te fastEquals topE) => {
+ val theTrait = te.as("theTrait")
+ val theInstance = theTrait.traitInstance().as("theInstance")
+ val outE =
+ theInstance.select(id("theTrait").as("traitDetails"),
+ id("theInstance").as("instanceInfo"))
+ QueryProcessor.validate(outE)
+ }
+ }
+
+ def typeTestExpression(typeName : String) : String = {
+ val stats = gPersistenceBehavior.typeTestExpression(typeName, counter)
+ preStatements ++= stats.init
+ stats.last
+ }
+
+ private def genQuery(expr: Expression, inSelect: Boolean): String = expr match {
+ case ClassExpression(clsName) =>
+ typeTestExpression(clsName)
+ case TraitExpression(clsName) =>
+ typeTestExpression(clsName)
+ case fe@FieldExpression(fieldName, fInfo, child) if fe.dataType.getTypeCategory == TypeCategory.PRIMITIVE => {
+ val fN = "\"" + gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo) + "\""
+ child match {
+ case Some(e) => s"${genQuery(e, inSelect)}.$fN"
+ case None => s"$fN"
+ }
+ }
+ case fe@FieldExpression(fieldName, fInfo, child)
+ if fe.dataType.getTypeCategory == TypeCategory.CLASS || fe.dataType.getTypeCategory == TypeCategory.STRUCT => {
+ val direction = if (fInfo.isReverse) "in" else "out"
+ val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo)
+ val step = s"""$direction("$edgeLbl")"""
+ child match {
+ case Some(e) => s"${genQuery(e, inSelect)}.$step"
+ case None => step
+ }
+ }
+ case fe@FieldExpression(fieldName, fInfo, child)
+ if fInfo.traitName != null => {
+ val direction = gPersistenceBehavior.instanceToTraitEdgeDirection
+ val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo)
+ val step = s"""$direction("$edgeLbl")"""
+ child match {
+ case Some(e) => s"${genQuery(e, inSelect)}.$step"
+ case None => step
+ }
+ }
+ case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => {
+ val fieldGremlinExpr = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}"
+ ch match {
+ case Some(child) => {
+ s"""${genQuery(child, inSelect)}.has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
+ }
+ case None => s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)"""
+ }
+ }
+ case fil@FilterExpression(child, condExpr) => {
+ s"${genQuery(child, inSelect)}.${genQuery(condExpr, inSelect)}"
+ }
+ case l@LogicalExpression(symb, children) => {
+ s"""$symb${children.map("_()." + genQuery(_, inSelect)).mkString("(", ",", ")")}"""
+ }
+ case sel@SelectExpression(child, selList) => {
+ val m = groupSelectExpressionsBySrc(sel)
+ var srcNamesList: List[String] = List()
+ var srcExprsList: List[List[String]] = List()
+ val it = m.iterator
+ while (it.hasNext) {
+ val (src, selExprs) = it.next
+ srcNamesList = srcNamesList :+ s""""$src""""
+ srcExprsList = srcExprsList :+ selExprs.map { selExpr =>
+ genQuery(selExpr, true)
+ }
+ }
+ val srcNamesString = srcNamesList.mkString("[", ",", "]")
+ val srcExprsStringList = srcExprsList.map {
+ _.mkString("[", ",", "]")
+ }
+ val srcExprsString = srcExprsStringList.foldLeft("")(_ + "{" + _ + "}")
+ s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString"
+ }
+ case loop@LoopExpression(input, loopExpr, t) => {
+ val inputQry = genQuery(input, inSelect)
+ val loopingPathGExpr = genQuery(loopExpr, inSelect)
+ val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")"""
+ val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{true}"
+ val loopObjectGExpr = gPersistenceBehavior.loopObjectExpression(input.dataType)
+ s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}"""
+ }
+ case BackReference(alias, _, _) =>
+ if (inSelect) gPersistenceBehavior.fieldPrefixInSelect else s"""back("$alias")"""
+ case AliasExpression(child, alias) => s"""${genQuery(child, inSelect)}.as("$alias")"""
+ case isTraitLeafExpression(traitName, Some(clsExp)) =>
+ s"""out("${gPersistenceBehavior.traitLabel(clsExp.dataType, traitName)}")"""
+ case isTraitUnaryExpression(traitName, child) =>
+ s"""out("${gPersistenceBehavior.traitLabel(child.dataType, traitName)}")"""
+ case hasFieldLeafExpression(fieldName, Some(clsExp)) =>
+ s"""has("$fieldName")"""
+ case hasFieldUnaryExpression(fieldName, child) =>
+ s"""${genQuery(child, inSelect)}.has("$fieldName")"""
+ case ArithmeticExpression(symb, left, right) => s"${genQuery(left, inSelect)} $symb ${genQuery(right, inSelect)}"
+ case l: Literal[_] => l.toString
+ case in@TraitInstanceExpression(child) => {
+ val direction = gPersistenceBehavior.traitToInstanceEdgeDirection
+ s"${genQuery(child, inSelect)}.$direction()"
+ }
+ case in@InstanceExpression(child) => {
+ s"${genQuery(child, inSelect)}"
+ }
+ case pe@PathExpression(child) => {
+ s"${genQuery(child, inSelect)}.path"
+ }
+ case x => throw new GremlinTranslationException(x, "expression not yet supported")
+ }
+
+ def genFullQuery(expr: Expression): String = {
+ var q = genQuery(expr, false)
+
+ if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) {
+ q = s"g.V.$q"
+ }
+
+ q = s"$q.toList()"
+
+ q = (preStatements ++ Seq(q) ++ postStatements).mkString("", ";", "")
+ /*
+ * the L:{} represents a groovy code block; the label is needed
+ * to distinguish it from a groovy closure.
+ */
+ s"L:{$q}"
+ }
+
+ def translate(): GremlinQuery = {
+ var e1 = expr.transformUp(wrapAndRule)
+
+ e1.traverseUp(validateComparisonForm)
+
+ e1 = e1.transformUp(new AddAliasToSelectInput)
+ e1.traverseUp(validateSelectExprHaveOneSrc)
+ e1 = e1.transformUp(addAliasToLoopInput())
+ e1 = e1.transformUp(instanceClauseToTop(e1))
+ e1 = e1.transformUp(traitClauseWithInstanceForTop(e1))
+
+ e1 match {
+ case e1: SelectExpression => {
+ val rMap = buildResultMapping(e1)
+ GremlinQuery(e1, genFullQuery(e1), rMap)
+ }
+ case pe@PathExpression(se@SelectExpression(child, selectList)) => {
+ val rMap = buildResultMapping(se)
+ GremlinQuery(e1, genFullQuery(e1), rMap)
+ }
+ case e1 => GremlinQuery(e1, genFullQuery(e1), null)
+ }
+
+ }
+
+ /*
+ * Translation Issues:
+ * 1. back references in filters. For e.g. testBackreference: 'DB as db Table where (db.name = "Reporting")'
+ * this is translated to:
+ * g.V.has("typeName","DB").as("db").in("Table.db").and(_().back("db").has("name", T.eq, "Reporting")).map().toList()
+ * But the '_().back("db") within the and is ignored, the has condition is applied on the current element.
+ * The solution is to to do predicate pushdown and apply the filter immediately on top of the referred Expression.
+ */
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala b/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala
new file mode 100755
index 0000000..5b129fb
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/QueryParser.scala
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import org.apache.atlas.query.Expressions._
+
+import scala.util.parsing.combinator.lexical.StdLexical
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.{ImplicitConversions, PackratParsers}
+import scala.util.parsing.input.CharArrayReader._
+
+trait QueryKeywords {
+ this: StandardTokenParsers =>
+
+ import scala.language.implicitConversions
+
+ protected case class Keyword(str: String)
+
+ protected implicit def asParser(k: Keyword): Parser[String] = k.str
+
+ protected val LPAREN = Keyword("(")
+ protected val RPAREN = Keyword(")")
+ protected val EQ = Keyword("=")
+ protected val LT = Keyword("<")
+ protected val GT = Keyword(">")
+ protected val NEQ = Keyword("!=")
+ protected val LTE = Keyword("<=")
+ protected val GTE = Keyword(">=")
+ protected val COMMA = Keyword(",")
+ protected val AND = Keyword("and")
+ protected val OR = Keyword("or")
+ protected val PLUS = Keyword("+")
+ protected val MINUS = Keyword("-")
+ protected val STAR = Keyword("*")
+ protected val DIV = Keyword("/")
+ protected val DOT = Keyword(".")
+
+ protected val SELECT = Keyword("select")
+ protected val FROM = Keyword("from")
+ protected val WHERE = Keyword("where")
+ protected val GROUPBY = Keyword("groupby")
+ protected val LOOP = Keyword("loop")
+ protected val ISA = Keyword("isa")
+ protected val IS = Keyword("is")
+ protected val HAS = Keyword("has")
+ protected val AS = Keyword("as")
+ protected val TIMES = Keyword("times")
+ protected val WITHPATH = Keyword("withPath")
+}
+
+trait ExpressionUtils {
+
+ def loop(input: Expression, l: (Expression, Option[Literal[Integer]], Option[String])) = l match {
+ case (c, None, None) => input.loop(c)
+ case (c, t, None) => input.loop(c, t.get)
+ case (c, None, Some(a)) => input.loop(c).as(a)
+ case (c, t, Some(a)) => input.loop(c, t.get).as(a)
+ }
+
+ def select(input: Expression, s: List[(Expression, Option[String])]) = {
+ val selList = s.map { t =>
+ t._2 match {
+ case None => t._1
+ case _ => t._1.as(t._2.get)
+ }
+ }
+ input.select(selList: _*)
+ }
+
+ def leftmostId(e: Expression) = {
+ var le: IdExpression = null
+ e.traverseUp { case i: IdExpression if le == null => le = i}
+ le
+ }
+
+ def notIdExpression = new PartialFunction[Expression, Expression] {
+ def isDefinedAt(x: Expression): Boolean = !x.isInstanceOf[IdExpression]
+
+ def apply(e: Expression) = e
+ }
+
+ def replaceIdWithField(id: IdExpression, fe: UnresolvedFieldExpression): PartialFunction[Expression, Expression] = {
+ case e: IdExpression if e == id => fe
+ }
+
+ def merge(snglQuery1: Expression, sngQuery2: Expression): Expression = {
+ val leftSrcId = leftmostId(sngQuery2)
+ sngQuery2.transformUp(replaceIdWithField(leftSrcId, snglQuery1.field(leftSrcId.name)))
+ }
+}
+
+class QueryParser extends StandardTokenParsers with QueryKeywords with ExpressionUtils with PackratParsers {
+
+ import scala.language.higherKinds
+
+ private val reservedWordsDelims: Seq[String] = this.
+ getClass.getMethods.filter(_.getReturnType == classOf[Keyword]).map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ private val (queryreservedWords: Seq[String], querydelims: Seq[String]) =
+ reservedWordsDelims.partition(s => s.charAt(0).isLetter)
+
+ override val lexical = new QueryLexer(queryreservedWords, querydelims)
+
+ def apply(input: String): Either[NoSuccess, Expression] = {
+ phrase(queryWithPath)(new lexical.Scanner(input)) match {
+ case Success(r, x) => Right(r)
+ case f@Failure(m, x) => Left(f)
+ case e@Error(m, x) => Left(e)
+ }
+ }
+
+ def queryWithPath = query ~ opt(WITHPATH) ^^ {
+ case q ~ None => q
+ case q ~ p => q.path()
+ }
+
+ def query: Parser[Expression] = rep1sep(singleQuery, opt(COMMA)) ^^ { l => l match {
+ case h :: Nil => h
+ case h :: t => t.foldLeft(h)(merge(_, _))
+ }
+ }
+
+ def singleQuery = singleQrySrc ~ opt(loopExpression) ~ opt(selectClause) ^^ {
+ case s ~ None ~ None => s
+ case s ~ l ~ None => loop(s, l.get)
+ case s ~ l ~ sel if l.isDefined => select(loop(s, l.get), sel.get)
+ case s ~ None ~ sel => select(s, sel.get)
+ }
+
+ /**
+ * A SingleQuerySrc can have the following forms:
+ * 1. FROM id [WHERE] [expr] -> from optionally followed by a filter
+ * 2. WHERE expr -> where clause, FROM is assumed to be the leftmost Id in the where clause
+ * 3. expr (that is not an IdExpression) -> where clause, FROM is assumed to be the leftmost Id in the expr
+ * 4. Id [WHERE] [expr] -> from optionally followed by a filter
+ *
+ * @return
+ */
+ def singleQrySrc: Parser[Expression] = FROM ~ fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ {
+ case f ~ i ~ w ~ None => i
+ case f ~ i ~ w ~ c => i.where(c.get)
+ } |
+ WHERE ~ (expr ^? notIdExpression) ^^ { case w ~ e => {
+ val lId = leftmostId(e)
+ if (lId == null) {
+ failure("cannot infer Input from the where clause")
+ }
+ lId.where(e)
+ }
+ } |
+ expr ^? notIdExpression ^^ { case e => {
+ val lId = leftmostId(e)
+ if (lId == null) {
+ failure("cannot infer Input from the where clause")
+ }
+ lId.where(e)
+ }
+ } |
+ fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ {
+ case i ~ w ~ None => i
+ case i ~ w ~ c => i.where(c.get)
+ }
+
+ def fromSrc = identifier ~ AS ~ alias ^^ { case s ~ a ~ al => s.as(al)} |
+ identifier
+
+
+ def loopExpression: Parser[(Expression, Option[Literal[Integer]], Option[String])] =
+ LOOP ~ (LPAREN ~> query <~ RPAREN) ~ opt(intConstant <~ TIMES) ~ opt(AS ~> alias) ^^ {
+ case l ~ e ~ None ~ a => (e, None, a)
+ case l ~ e ~ Some(i) ~ a => (e, Some(int(i)), a)
+ }
+
+ def selectClause: Parser[List[(Expression, Option[String])]] = SELECT ~ rep1sep(selectExpression, COMMA) ^^ {
+ case s ~ cs => cs
+ }
+
+ def selectExpression: Parser[(Expression, Option[String])] = expr ~ opt(AS ~> alias) ^^ {
+ case e ~ a => (e, a)
+ }
+
+ def expr: Parser[Expression] = compE ~ opt(rep(exprRight)) ^^ {
+ case l ~ None => l
+ case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.logicalOp(r._1)(r._2)}
+ }
+
+ def exprRight = (AND | OR) ~ compE ^^ { case op ~ c => (op, c)}
+
+ def compE =
+ arithE ~ (LT | LTE | EQ | NEQ | GT | GTE) ~ arithE ^^ { case l ~ op ~ r => l.compareOp(op)(r)} |
+ arithE ~ (ISA | IS) ~ ident ^^ { case l ~ i ~ t => l.isTrait(t)} |
+ arithE ~ HAS ~ ident ^^ { case l ~ i ~ f => l.hasField(f)} |
+ arithE
+
+ def arithE = multiE ~ opt(rep(arithERight)) ^^ {
+ case l ~ None => l
+ case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)}
+ }
+
+ def arithERight = (PLUS | MINUS) ~ multiE ^^ { case op ~ r => (op, r)}
+
+ def multiE = atomE ~ opt(rep(multiERight)) ^^ {
+ case l ~ None => l
+ case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)}
+ }
+
+ def multiERight = (STAR | DIV) ~ atomE ^^ { case op ~ r => (op, r)}
+
+
+ def atomE = literal | identifier | LPAREN ~> expr <~ RPAREN
+
+ def identifier = rep1sep(ident, DOT) ^^ { l => l match {
+ case h :: Nil => id(h)
+ case h :: t => {
+ t.foldLeft(id(h).asInstanceOf[Expression])(_.field(_))
+ }
+ }
+ }
+
+ def alias = ident | stringLit
+
+ def literal = booleanConstant ^^ {
+ boolean(_)
+ } |
+ intConstant ^^ {
+ int(_)
+ } |
+ longConstant ^^ {
+ long(_)
+ } |
+ floatConstant ^^ {
+ float(_)
+ } |
+ doubleConstant ^^ {
+ double(_)
+ } |
+ stringLit ^^ {
+ string(_)
+ }
+
+ def booleanConstant: Parser[String] =
+ elem("int", _.isInstanceOf[lexical.BooleanLiteral]) ^^ (_.chars)
+
+ def intConstant: Parser[String] =
+ elem("int", _.isInstanceOf[lexical.IntLiteral]) ^^ (_.chars)
+
+ def longConstant: Parser[String] =
+ elem("int", _.isInstanceOf[lexical.LongLiteral]) ^^ (_.chars)
+
+ def floatConstant: Parser[String] =
+ elem("int", _.isInstanceOf[lexical.FloatLiteral]) ^^ (_.chars)
+
+ def doubleConstant: Parser[String] =
+ elem("int", _.isInstanceOf[lexical.DoubleLiteral]) ^^ (_.chars)
+
+}
+
+class QueryLexer(val keywords: Seq[String], val delims: Seq[String]) extends StdLexical with ImplicitConversions {
+
+ case class BooleanLiteral(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ case class IntLiteral(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ case class LongLiteral(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ case class FloatLiteral(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ case class DoubleLiteral(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ reserved ++= keywords.flatMap(w => allCaseVersions(w))
+
+ delimiters ++= delims
+
+ override lazy val token: Parser[Token] =
+ (
+ (trueP | falseP)
+ | longConstant ^^ LongLiteral
+ | intConstant ^^ IntLiteral
+ | floatConstant ^^ FloatLiteral
+ | dubConstant ^^ DoubleLiteral
+ | identifier ^^ processIdent
+ | string ^^ StringLit
+ | EofCh ^^^ EOF
+ | '\'' ~> failure("unclosed string literal")
+ | '"' ~> failure("unclosed string literal")
+ | delim
+ | '.' ^^^ new Keyword(".")
+ | failure("illegal character")
+ )
+
+ override def identChar = letter | elem('_')
+
+ def identifier = identChar ~ (identChar | digit).* ^^ { case first ~ rest => (first :: rest).mkString} |
+ '`' ~> chrExcept('`', '\n', EofCh).* <~ '`' ^^ {
+ _ mkString ""
+ }
+
+ override def whitespace: Parser[Any] =
+ (whitespaceChar
+ | '/' ~ '*' ~ comment
+ | '/' ~ '/' ~ chrExcept(EofCh, '\n').*
+ | '#' ~ chrExcept(EofCh, '\n').*
+ | '/' ~ '*' ~ failure("unclosed comment")
+ ).*
+
+ protected override def comment: Parser[Any] = (
+ commentChar.* ~ '*' ~ '/'
+ )
+
+ protected def commentChar = chrExcept(EofCh, '*') | '*' ~ not('/')
+
+ def string = '\"' ~> chrExcept('\"', '\n', EofCh).* <~ '\"' ^^ {
+ _ mkString ""
+ } |
+ '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^ {
+ _ mkString ""
+ }
+
+ def zero: Parser[String] = '0' ^^^ "0"
+
+ def nonzero = elem("nonzero digit", d => d.isDigit && d != '0')
+
+ def sign = elem("sign character", d => d == '-' || d == '+')
+
+ def exponent = elem("exponent character", d => d == 'e' || d == 'E')
+
+
+ def intConstant = opt(sign) ~> zero | intList
+
+ def intList = opt(sign) ~ nonzero ~ rep(digit) ^^ { case s ~ x ~ y => (optString("", s) :: x :: y) mkString ""}
+
+ def fracPart: Parser[String] = '.' ~> rep(digit) ^^ { r =>
+ "." + (r mkString "")
+ }
+
+ def expPart = exponent ~ opt(sign) ~ rep1(digit) ^^ { case e ~ s ~ d =>
+ e.toString + optString("", s) + d.mkString("")
+ }
+
+ def dubConstant = opt(sign) ~ digit.+ ~ fracPart ~ opt(expPart) ^^ {
+ case s ~ i ~ f ~ e => {
+ optString("", s) + (i mkString "") + f + optString("", e)
+ }
+ }
+
+ def floatConstant = opt(sign) ~ digit.* ~ fracPart ~ 'f' ^^ { case s ~ i ~ fr ~ f =>
+ optString("", s) + i + fr
+ } | opt(sign) ~ digit.+ ~ opt(fracPart) ~ 'f' ^^ { case s ~ i ~ fr ~ f =>
+ optString("", s) + i + optString("", fr)
+ }
+
+ def longConstant = intConstant ~ 'l' ^^ { case i ~ l => i}
+
+ def trueP = 't' ~ 'r' ~ 'u' ~ 'e' ^^^ BooleanLiteral("true")
+
+ def falseP = 'f' ~ 'a' ~ 'l' ~ 's' ~ 'e' ^^^ BooleanLiteral("false")
+
+ private def optString[A](pre: String, a: Option[A]) = a match {
+ case Some(x) => pre + x.toString
+ case None => ""
+ }
+
+ /** Generate all variations of upper and lower case of a given string */
+ def allCaseVersions(s: String, prefix: String = ""): Stream[String] = {
+ if (s.isEmpty) {
+ Stream(prefix)
+ } else {
+ allCaseVersions(s.tail, prefix + s.head.toLower) #:::
+ allCaseVersions(s.tail, prefix + s.head.toUpper)
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala b/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala
new file mode 100755
index 0000000..0d2a908
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import com.thinkaurelius.titan.core.TitanGraph
+import org.apache.atlas.query.Expressions._
+import org.slf4j.{Logger, LoggerFactory}
+
+object QueryProcessor {
+ val LOG : Logger = LoggerFactory.getLogger("org.apache.atlas.query.QueryProcessor")
+
+ def evaluate(e: Expression, g: TitanGraph, gP : GraphPersistenceStrategies = GraphPersistenceStrategy1):
+ GremlinQueryResult = {
+ val e1 = validate(e)
+ val q = new GremlinTranslator(e1, gP).translate()
+ LOG.debug("Query: " + e1)
+ LOG.debug("Expression Tree:\n" + e1.treeString)
+ LOG.debug("Gremlin Query: " + q.queryStr)
+ new GremlinEvaluator(q, gP, g).evaluate()
+ }
+
+ def validate(e: Expression): Expression = {
+ val e1 = e.transformUp(new Resolver())
+
+ e1.traverseUp {
+ case x: Expression if !x.resolved =>
+ throw new ExpressionException(x, s"Failed to resolved expression $x")
+ }
+
+ /*
+ * trigger computation of dataType of expression tree
+ */
+ e1.dataType
+
+ /*
+ * ensure fieldReferences match the input expression's dataType
+ */
+ val e2 = e1.transformUp(FieldValidator)
+ val e3 = e2.transformUp(new Resolver())
+
+ e3.dataType
+
+ e3
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/Resolver.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/Resolver.scala b/repository/src/main/scala/org/apache/atlas/query/Resolver.scala
new file mode 100755
index 0000000..142ba8d
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/Resolver.scala
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import org.apache.atlas.query.Expressions._
+import org.apache.atlas.typesystem.types.IDataType
+
+class Resolver(srcExpr: Option[Expression] = None, aliases: Map[String, Expression] = Map(),
+ connectClassExprToSrc: Boolean = false)
+ extends PartialFunction[Expression, Expression] {
+
+ import org.apache.atlas.query.TypeUtils._
+
+ def isDefinedAt(x: Expression) = true
+
+ def apply(e: Expression): Expression = e match {
+ case idE@IdExpression(name) => {
+ val backExpr = aliases.get(name)
+ if (backExpr.isDefined) {
+ return new BackReference(name, backExpr.get, None)
+ }
+ if (srcExpr.isDefined) {
+ val fInfo = resolveReference(srcExpr.get.dataType, name)
+ if (fInfo.isDefined) {
+ return new FieldExpression(name, fInfo.get, None)
+ }
+ }
+ val cType = resolveAsClassType(name)
+ if (cType.isDefined) {
+ return new ClassExpression(name)
+ }
+ val tType = resolveAsTraitType(name)
+ if (tType.isDefined) {
+ return new TraitExpression(name)
+ }
+ idE
+ }
+ case ce@ClassExpression(clsName) if connectClassExprToSrc && srcExpr.isDefined => {
+ val fInfo = resolveReference(srcExpr.get.dataType, clsName)
+ if (fInfo.isDefined) {
+ return new FieldExpression(clsName, fInfo.get, None)
+ }
+ ce
+ }
+ case f@UnresolvedFieldExpression(child, fieldName) if child.resolved => {
+ var fInfo: Option[FieldInfo] = None
+
+ fInfo = resolveReference(child.dataType, fieldName)
+ if (fInfo.isDefined) {
+ return new FieldExpression(fieldName, fInfo.get, Some(child))
+ }
+ val tType = resolveAsTraitType(fieldName)
+ if (tType.isDefined) {
+ return new FieldExpression(fieldName, FieldInfo(child.dataType, null, null, fieldName), Some(child))
+ }
+ f
+ }
+ case isTraitLeafExpression(traitName, classExpression)
+ if srcExpr.isDefined && !classExpression.isDefined =>
+ isTraitLeafExpression(traitName, srcExpr)
+ case hasFieldLeafExpression(traitName, classExpression)
+ if srcExpr.isDefined && !classExpression.isDefined =>
+ hasFieldLeafExpression(traitName, srcExpr)
+ case f@FilterExpression(inputExpr, condExpr) if inputExpr.resolved => {
+ val r = new Resolver(Some(inputExpr), inputExpr.namedExpressions)
+ return new FilterExpression(inputExpr, condExpr.transformUp(r))
+ }
+ case SelectExpression(child, selectList) if child.resolved => {
+ val r = new Resolver(Some(child), child.namedExpressions)
+ return new SelectExpression(child, selectList.map {
+ _.transformUp(r)
+ })
+ }
+ case l@LoopExpression(inputExpr, loopExpr, t) if inputExpr.resolved => {
+ val r = new Resolver(Some(inputExpr), inputExpr.namedExpressions, true)
+ return new LoopExpression(inputExpr, loopExpr.transformUp(r), t)
+ }
+ case x => x
+ }
+}
+
+/**
+ * - any FieldReferences that explicitly reference the input, can be converted to implicit references
+ * - any FieldReferences that explicitly reference a
+ */
+object FieldValidator extends PartialFunction[Expression, Expression] {
+
+ def isDefinedAt(x: Expression) = true
+
+ def isSrc(e: Expression) = e.isInstanceOf[ClassExpression] || e.isInstanceOf[TraitExpression]
+
+ def validateQualifiedField(srcDataType: IDataType[_]): PartialFunction[Expression, Expression] = {
+ case FieldExpression(fNm, fInfo, Some(child))
+ if (child.children == Nil && !child.isInstanceOf[BackReference] && child.dataType == srcDataType) =>
+ FieldExpression(fNm, fInfo, None)
+ case fe@FieldExpression(fNm, fInfo, Some(child)) if isSrc(child) =>
+ throw new ExpressionException(fe, s"srcType of field doesn't match input type")
+ case hasFieldUnaryExpression(fNm, child) if child.dataType == srcDataType =>
+ hasFieldLeafExpression(fNm)
+ case hF@hasFieldUnaryExpression(fNm, child) if isSrc(child) =>
+ throw new ExpressionException(hF, s"srcType of field doesn't match input type")
+ case isTraitUnaryExpression(fNm, child) if child.dataType == srcDataType =>
+ isTraitLeafExpression(fNm)
+ case iT@isTraitUnaryExpression(fNm, child) if isSrc(child) =>
+ throw new ExpressionException(iT, s"srcType of field doesn't match input type")
+ }
+
+ def validateOnlyFieldReferencesInLoopExpressions(loopExpr: LoopExpression)
+ : PartialFunction[Expression, Unit] = {
+ case f: FieldExpression => ()
+ case x => throw new ExpressionException(loopExpr,
+ s"Loop Expression can only contain field references; '${x.toString}' not supported.")
+ }
+
+ def apply(e: Expression): Expression = e match {
+ case f@FilterExpression(inputExpr, condExpr) => {
+ val validatedCE = condExpr.transformUp(validateQualifiedField(inputExpr.dataType))
+ if (validatedCE.fastEquals(condExpr)) {
+ f
+ } else {
+ new FilterExpression(inputExpr, validatedCE)
+ }
+ }
+ case SelectExpression(child, selectList) if child.resolved => {
+ val v = validateQualifiedField(child.dataType)
+ return new SelectExpression(child, selectList.map {
+ _.transformUp(v)
+ })
+ }
+ case l@LoopExpression(inputExpr, loopExpr, t) => {
+ val validatedLE = loopExpr.transformUp(validateQualifiedField(inputExpr.dataType))
+ val l1 = {
+ if (validatedLE.fastEquals(loopExpr)) l
+ else new LoopExpression(inputExpr, validatedLE, t)
+ }
+ l1.loopingExpression.traverseUp(validateOnlyFieldReferencesInLoopExpressions(l1))
+ l1
+ }
+ case x => x
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
new file mode 100755
index 0000000..4212a34
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import java.util
+import java.util.concurrent.atomic.AtomicInteger
+
+import org.apache.atlas.MetadataException
+import org.apache.atlas.query.Expressions.{PathExpression, SelectExpression}
+import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
+import org.apache.atlas.typesystem.types._
+
+object TypeUtils {
+ val typSystem = TypeSystem.getInstance()
+
+ def numericTypes : Seq[PrimitiveType[_]] = Seq(DataTypes.BYTE_TYPE,
+ DataTypes.SHORT_TYPE,
+ DataTypes.INT_TYPE,
+ DataTypes.FLOAT_TYPE,
+ DataTypes.LONG_TYPE,
+ DataTypes.DOUBLE_TYPE,
+ DataTypes.BIGINTEGER_TYPE,
+ DataTypes.BIGDECIMAL_TYPE)
+
+ def combinedType(typ1 : IDataType[_], typ2 : IDataType[_]) : PrimitiveType[_] = {
+ val typ1Idx = if (numericTypes.contains(typ1)) Some(numericTypes.indexOf(typ1)) else None
+ val typ2Idx = if (numericTypes.contains(typ2)) Some(numericTypes.indexOf(typ2)) else None
+
+ if ( typ1Idx.isDefined && typ2Idx.isDefined ) {
+ val rIdx = math.max(typ1Idx.get, typ2Idx.get)
+
+ if ( (typ1 == DataTypes.FLOAT_TYPE && typ2 == DataTypes.LONG_TYPE) ||
+ (typ1 == DataTypes.LONG_TYPE && typ2 == DataTypes.FLOAT_TYPE) ) {
+ return DataTypes.DOUBLE_TYPE
+ }
+ return numericTypes(rIdx)
+ }
+
+ throw new MetadataException(s"Cannot combine types: ${typ1.getName} and ${typ2.getName}")
+ }
+
+ var tempStructCounter : AtomicInteger = new AtomicInteger(0)
+ val TEMP_STRUCT_NAME_PREFIX = "__tempQueryResultStruct"
+ def createStructType(selectExprs : List[Expressions.AliasExpression]) : StructType = {
+ val aDefs = new Array[AttributeDefinition](selectExprs.size)
+ selectExprs.zipWithIndex.foreach { t =>
+ val (e,i) = t
+ aDefs(i) = new AttributeDefinition(e.alias,e.dataType.getName, Multiplicity.OPTIONAL, false, null)
+ }
+ return typSystem.defineQueryResultType(s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}",
+ null,
+ aDefs:_*);
+ }
+
+ object ResultWithPathStruct {
+ val pathAttrName = "path"
+ val resultAttrName = "result"
+ val pathAttrType = DataTypes.arrayTypeName(typSystem.getIdType.getStructType)
+
+ val pathAttr = new AttributeDefinition(pathAttrName, pathAttrType, Multiplicity.COLLECTION, false, null)
+
+ def createType(pE : PathExpression, resultType : IDataType[_]) : StructType = {
+ val resultAttr = new AttributeDefinition(resultAttrName, resultType.getName, Multiplicity.REQUIRED, false, null)
+ val typName = s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}"
+ val m : java.util.HashMap[String, IDataType[_]] = new util.HashMap[String, IDataType[_]]()
+ if ( pE.child.isInstanceOf[SelectExpression]) {
+ m.put(pE.child.dataType.getName, pE.child.dataType)
+ }
+ typSystem.defineQueryResultType(typName, m, pathAttr, resultAttr);
+ }
+ }
+
+ /**
+ * Structure representing the Closure Graph.
+ * Returns:
+ * 1. A map of vertexId -> vertex Info(these are the attributes requested in the query)
+ * 2. A edges map: each entry is a mapping from an vertexId to the List of adjacent vertexIds.
+ *
+ * '''The Vertex Map doesn't contain all the vertices in the Graph. Only the ones for which Attributes are
+ * available.''' These are the vertices that represent the EntityType whose Closure was requested. For e.g. for
+ * Table Lineage the ''vertex map'' will contain information about Tables, but not about ''Load Process'' vertices
+ * that connect Tables.
+ */
+ object GraphResultStruct {
+ val SRC_PREFIX = "src"
+ val DEST_PREFIX = "dest"
+
+ val verticesAttrName = "vertices"
+ val edgesAttrName = "edges"
+ val vertexIdAttrName = "vertexId"
+
+ lazy val edgesAttrType = typSystem.defineMapType(DataTypes.STRING_TYPE,
+ typSystem.defineArrayType(DataTypes.STRING_TYPE))
+
+ def createType(resultWithPathType: StructType): StructType = {
+ val resultType = resultWithPathType.fieldMapping().fields.get(ResultWithPathStruct.resultAttrName).dataType()
+
+ val verticesAttrType = typSystem.defineMapType(DataTypes.STRING_TYPE,
+ vertexType(resultType.asInstanceOf[StructType]))
+ val typName = s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}"
+ val verticesAttr = new AttributeDefinition(verticesAttrName, verticesAttrType.getName,
+ Multiplicity.REQUIRED, false, null)
+ val edgesAttr = new AttributeDefinition(edgesAttrName, edgesAttrType.getName, Multiplicity.REQUIRED, false, null)
+
+ val m: java.util.HashMap[String, IDataType[_]] = new util.HashMap[String, IDataType[_]]()
+ m.put(resultWithPathType.getName, resultWithPathType)
+ m.put(resultType.getName, resultType)
+ m.put(edgesAttrType.getName, edgesAttrType)
+ m.put(verticesAttrType.getName, verticesAttrType)
+ typSystem.defineQueryResultType(typName, m, verticesAttr, edgesAttr)
+ }
+
+ private def vertexType(resultType: StructType): StructType = {
+
+ import scala.collection.JavaConverters._
+
+ var attrs: List[AttributeDefinition] =
+ resultType.fieldMapping.fields.asScala.filter(_._1.startsWith(s"${SRC_PREFIX}_")).mapValues { aInfo =>
+
+ new AttributeDefinition(aInfo.name.substring(s"${SRC_PREFIX}_".length), aInfo.dataType.getName,
+ aInfo.multiplicity, aInfo.isComposite, aInfo.reverseAttributeName)
+ }.values.toList
+
+ attrs = new AttributeDefinition(vertexIdAttrName, typSystem.getIdType.getStructType.name,
+ Multiplicity.REQUIRED, false, null) :: attrs
+
+ return typSystem.defineQueryResultType(s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}",
+ null,
+ attrs: _*)
+ }
+ }
+
+ def fieldMapping(iDataType: IDataType[_]) : Option[FieldMapping] = iDataType match {
+ case c : ClassType => Some(c.fieldMapping())
+ case t : TraitType => Some(t.fieldMapping())
+ case s : StructType => Some(s.fieldMapping())
+ case _ => None
+ }
+
+ def hasFields(iDataType: IDataType[_]) : Boolean = {
+ fieldMapping(iDataType).isDefined
+ }
+
+ import scala.language.existentials
+ case class FieldInfo(dataType : IDataType[_],
+ attrInfo : AttributeInfo,
+ reverseDataType : IDataType[_] = null,
+ traitName : String = null) {
+ def isReverse = reverseDataType != null
+ override def toString : String = {
+ if ( traitName != null ) {
+ s"""FieldInfo("${dataType.getName}", "$traitName")"""
+ }
+ else if ( reverseDataType == null ) {
+ s"""FieldInfo("${dataType.getName}", "${attrInfo.name}")"""
+ } else {
+ s"""FieldInfo("${dataType.getName}", "${attrInfo.name}", "${reverseDataType.getName}")"""
+ }
+ }
+ }
+
+ val FIELD_QUALIFIER = "(.*?)(->.*)?".r
+
+ /**
+ * Given a ComposedType `t` and a name resolve using the following rules:
+ * - if `id` is a field in `t` resolve to the field
+ * - if `id` is the name of a Struct|Class|Trait Type and it has a field that is of type `t` then return that type
+ *
+ * For e.g.
+ * 1. if we have types Table(name : String, cols : List[Column]), Column(name : String) then
+ * `resolveReference(Table, "cols")` resolves to type Column. So a query can be "Table.cols"
+ * 2. But if we have Table(name : String), Column(name : String, tbl : Table) then "Table.Column" will resolve
+ * to type Column
+ *
+ * This way the language will support navigation even if the relationship is one-sided.
+ *
+ * @param typ
+ * @param id
+ * @return
+ */
+ def resolveReference(typ : IDataType[_], id : String) : Option[FieldInfo] = {
+
+ val fMap = fieldMapping(typ)
+ if ( fMap.isDefined) {
+
+ if (fMap.get.fields.containsKey(id)) {
+ return Some(FieldInfo(typ,fMap.get.fields.get(id)))
+ }
+
+ try {
+ val FIELD_QUALIFIER(clsNm, rest) = id
+ val idTyp = typSystem.getDataType(classOf[IDataType[_]], clsNm)
+ val idTypFMap = fieldMapping(idTyp)
+
+ if (rest != null ) {
+ val attrNm = rest.substring(2)
+
+ if (idTypFMap.get.fields.containsKey(attrNm)) {
+ return Some(FieldInfo(typ,idTypFMap.get.fields.get(attrNm), idTyp))
+ }
+ }
+
+ if (idTypFMap.isDefined) {
+ import scala.collection.JavaConversions._
+ val fields: Seq[AttributeInfo] = idTypFMap.get.fields.values().filter { aInfo =>
+ aInfo.dataType() == typ ||
+ ( aInfo.dataType().getTypeCategory == TypeCategory.ARRAY &&
+ aInfo.dataType().asInstanceOf[ArrayType].getElemType == typ
+ )
+ }.toSeq
+ if (fields.size == 1) {
+ return Some(FieldInfo(typ, fields(0), idTyp))
+ }
+ /*
+ * is there only 1 array field of this type?
+ * If yes resolve to it.
+ * @todo: allow user to specify the relationship to follow by further qualifying the type. for e.g.
+ * field("LoadProcess.inputTables")
+ */
+ val aFields = fields.filter { aInfo => aInfo.dataType().getTypeCategory == TypeCategory.ARRAY}
+ if (aFields.size == 1) {
+ return Some(FieldInfo(typ, aFields(0), idTyp))
+ }
+ }
+ } catch {
+ case _ : MetadataException => None
+ }
+ }
+ None
+ }
+
+ def resolveAsClassType(id : String) : Option[ClassType] = {
+ try {
+ Some(typSystem.getDataType(classOf[ClassType], id))
+ } catch {
+ case _ : MetadataException => None
+ }
+ }
+
+ def resolveAsTraitType(id : String) : Option[TraitType] = {
+ try {
+ Some(typSystem.getDataType(classOf[TraitType], id))
+ } catch {
+ case _ : MetadataException => None
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/ClosureQuery.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/ClosureQuery.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/ClosureQuery.scala
deleted file mode 100755
index 7ebbf64..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/ClosureQuery.scala
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import java.util
-
-import Expressions._
-import com.thinkaurelius.titan.core.TitanGraph
-import org.apache.hadoop.metadata.MetadataException
-import org.apache.hadoop.metadata.typesystem.ITypedStruct
-import org.apache.hadoop.metadata.typesystem.json.{InstanceSerialization, Serialization}
-import org.apache.hadoop.metadata.typesystem.persistence.{Id, StructInstance}
-import org.apache.hadoop.metadata.typesystem.types.{TypeSystem, StructType, DataTypes}
-import org.apache.hadoop.metadata.typesystem.types.DataTypes.{MapType, PrimitiveType}
-
-/**
- * Represents a Query to compute the closure based on a relationship between entities of a particular type.
- * For e.g. Database Tables are related to each other to capture the '''Lineage''' of data in a Table based
- * on other Tables.
- *
- * A Closure Query is specified by the following information:
- * - The Type whose instances are in a closure relationship. For e.g. 'Table'
- * - The Closure relation. This is specified as an ''Attribute path''. For e.g. if we have the following model:
- * {{{
- * class Table {
- * name : String,
- * ...
- * }
- *
- * class LoadTableProcess {
- * name : String,
- * inputTables : List[Table],
- * outputTable : Table,
- * ...
- * }
- * }}}
- * ''LoadTable'' instance captures the relationship between the data in an output Table and a set of input Tables.
- * In order to compute the '''Lineage''' of a Table, the ''Attribute path'' that relates 2 Tables is
- * '''[(LoadTableProcess,outputTable), inputTables]'''. This list is saying that for any Table I want to connect to other
- * tables via the LoadProcess.outputTable attribute, and then via the inputTables attribute. So each entry in the
- * Attribute Path represents an attribute in an object. For reverse relations the Type and attribute must be specified,
- * as in 'LoadTableProcess,outputTable)', whereas for forward relations the attribute name is sufficient.
- * - The depth of the traversal. Certain times you are not interested in the complete closure, but to only
- * discover related instances up to a certain depth. Specify the depth as number of hops, or you can ask for the
- * complete closure.
- * - You can ask for certain attributes to be returned. For e.g. you may only want the Table name, owner and
- * creationDate. By default only the Ids of the related instances is returned.
- * - For pair of related instances, you optionally ask for the Path of the relation to be returned. This is
- * returned as a list of ''Id''s.
- *
- * Given these 5 things the ClosureQuery can be executed, it returns a GremlinQueryResult of the Closure Query.
- */
-trait ClosureQuery {
-
- val SRC_PREFIX = TypeUtils.GraphResultStruct.SRC_PREFIX
- val DEST_PREFIX = TypeUtils.GraphResultStruct.DEST_PREFIX
-
- sealed trait PathAttribute {
-
- def toExpr : Expression = this match {
- case r : Relation => id(r.attributeName)
- case rr : ReverseRelation => id(s"${rr.typeName}->${rr.attributeName}")
- }
-
- def toFieldName : String = this match {
- case r : Relation => r.attributeName
- case rr : ReverseRelation => rr.typeName
- }
- }
- case class ReverseRelation(typeName : String, attributeName : String) extends PathAttribute
- case class Relation(attributeName : String) extends PathAttribute
-
- /**
- * Type on whose instances the closure needs to be computed
- * @return
- */
- def closureType : String
-
- /**
- * specify how instances are related.
- */
- def closureRelation : List[PathAttribute]
-
- /**
- * The maximum hops between related instances. A [[None]] implies there is maximum.
- * @return
- */
- def depth : Option[Int]
-
- /**
- * The attributes to return for the instances. These will be prefixed by 'src_' and 'dest_' in the
- * output rows.
- * @return
- */
- def selectAttributes : Option[List[String]]
-
- /**
- * specify if the Path should be returned.
- * @return
- */
- def withPath : Boolean
-
- def persistenceStrategy: GraphPersistenceStrategies
- def g: TitanGraph
-
- def pathExpr : Expressions.Expression = {
- closureRelation.tail.foldLeft(closureRelation.head.toExpr)((b,a) => b.field(a.toFieldName))
- }
-
- def selectExpr(alias : String) : List[Expression] = {
- selectAttributes.map { _.map { a =>
- id(alias).field(a).as(s"${alias}_$a")
- }
- }.getOrElse(List(id(alias)))
- }
-
- /**
- * hook to allow a filter to be added for the closureType
- * @param expr
- * @return
- */
- def srcCondition(expr : Expression) : Expression = expr
-
- def expr : Expressions.Expression = {
- val e = srcCondition(Expressions._class(closureType)).as(SRC_PREFIX).loop(pathExpr).as(DEST_PREFIX).
- select((selectExpr(SRC_PREFIX) ++ selectExpr(DEST_PREFIX)):_*)
- if (withPath) e.path else e
- }
-
- def evaluate(): GremlinQueryResult = {
- var e = expr
- QueryProcessor.evaluate(e, g, persistenceStrategy)
- }
-
- def graph : GraphResult = {
-
- if (!withPath) {
- throw new ExpressionException(expr, "Graph requested for non Path Query")
- }
-
- import scala.collection.JavaConverters._
-
- val res = evaluate()
-
- val graphResType = TypeUtils.GraphResultStruct.createType(res.resultDataType.asInstanceOf[StructType])
- val vertexPayloadType = {
- val mT = graphResType.fieldMapping.fields.get(TypeUtils.GraphResultStruct.verticesAttrName).
- dataType().asInstanceOf[MapType]
- mT.getValueType.asInstanceOf[StructType]
- }
-
- def id(idObj : StructInstance) : String = idObj.getString(TypeSystem.ID_STRUCT_ID_ATTRNAME)
-
- def vertexStruct(idObj : StructInstance, resRow : ITypedStruct, attrPrefix : String) : StructInstance = {
- val vP = vertexPayloadType.createInstance()
- vP.set(TypeUtils.GraphResultStruct.vertexIdAttrName, idObj)
- vertexPayloadType.fieldMapping.fields.asScala.keys.
- filter(_ != TypeUtils.GraphResultStruct.vertexIdAttrName).foreach{a =>
- vP.set(a, resRow.get(s"${attrPrefix}$a"))
- }
- vP.asInstanceOf[StructInstance]
- }
-
- val instance = graphResType.createInstance()
- val vertices = new util.HashMap[String, AnyRef]()
- val edges = new util.HashMap[String,java.util.List[String]]()
-
- /**
- * foreach resultRow
- * for each Path entry
- * add an entry in the edges Map
- * add an entry for the Src Vertex to the vertex Map
- * add an entry for the Dest Vertex to the vertex Map
- */
- res.rows.map(_.asInstanceOf[StructInstance]).foreach { r =>
-
- val path = r.get(TypeUtils.ResultWithPathStruct.pathAttrName).asInstanceOf[java.util.List[_]].asScala
- val srcVertex = path.head.asInstanceOf[StructInstance]
-
- var currVertex = srcVertex
- path.tail.foreach { n =>
- val nextVertex = n.asInstanceOf[StructInstance]
- val iList = if (!edges.containsKey(id(currVertex))) {
- val l = new util.ArrayList[String]()
- edges.put(id(currVertex), l)
- l
- } else {
- edges.get(id(currVertex))
- }
- if ( !iList.contains(id(nextVertex))) {
- iList.add(id(nextVertex))
- }
- currVertex = nextVertex
- }
- val vertex = r.get(TypeUtils.ResultWithPathStruct.resultAttrName)
- vertices.put(id(srcVertex), vertexStruct(srcVertex,
- r.get(TypeUtils.ResultWithPathStruct.resultAttrName).asInstanceOf[ITypedStruct],
- s"${SRC_PREFIX}_"))
- vertices.put(id(currVertex), vertexStruct(currVertex,
- r.get(TypeUtils.ResultWithPathStruct.resultAttrName).asInstanceOf[ITypedStruct],
- s"${DEST_PREFIX}_"))
- }
-
- instance.set(TypeUtils.GraphResultStruct.verticesAttrName, vertices)
- instance.set(TypeUtils.GraphResultStruct.edgesAttrName, edges)
- GraphResult(res.query, instance)
- }
-}
-
-/**
- * Closure for a single instance. Instance is specified by an ''attributeToSelectInstance'' and the value
- * for the attribute.
- *
- * @tparam T
- */
-trait SingleInstanceClosureQuery[T] extends ClosureQuery {
-
- def attributeToSelectInstance : String
-
- def attributeTyp : PrimitiveType[T]
- def instanceValue : T
-
- override def srcCondition(expr : Expression) : Expression = {
- expr.where(
- Expressions.id(attributeToSelectInstance).`=`(Expressions.literal(attributeTyp, instanceValue))
- )
- }
-}
-
-/**
- * A ClosureQuery to compute '''Lineage''' for Hive tables. Assumes the Lineage relation is captured in a ''CTAS''
- * type, and the table relations are captured as attributes from a CTAS instance to Table instances.
- *
- * @param tableTypeName The name of the Table Type.
- * @param ctasTypeName The name of the Create Table As Select(CTAS) Type.
- * @param ctasInputTableAttribute The attribute in CTAS Type that associates it to the ''Input'' tables.
- * @param ctasOutputTableAttribute The attribute in CTAS Type that associates it to the ''Output'' tables.
- * @param depth depth as needed by the closure Query.
- * @param selectAttributes as needed by the closure Query.
- * @param withPath as needed by the closure Query.
- * @param persistenceStrategy as needed to evaluate the Closure Query.
- * @param g as needed to evaluate the Closure Query.
- */
-case class HiveLineageQuery(tableTypeName : String,
- tableName : String,
- ctasTypeName : String,
- ctasInputTableAttribute : String,
- ctasOutputTableAttribute : String,
- depth : Option[Int],
- selectAttributes : Option[List[String]],
- withPath : Boolean,
- persistenceStrategy: GraphPersistenceStrategies,
- g: TitanGraph
- ) extends SingleInstanceClosureQuery[String] {
-
- val closureType : String = tableTypeName
-
- val attributeToSelectInstance = "name"
- val attributeTyp = DataTypes.STRING_TYPE
-
- val instanceValue = tableName
-
- lazy val closureRelation = List(
- ReverseRelation(ctasTypeName, ctasOutputTableAttribute),
- Relation(ctasInputTableAttribute)
- )
-}
-
-/**
- * A ClosureQuery to compute where a table is used based on the '''Lineage''' for Hive tables.
- * Assumes the Lineage relation is captured in a ''CTAS''
- * type, and the table relations are captured as attributes from a CTAS instance to Table instances.
- *
- * @param tableTypeName The name of the Table Type.
- * @param ctasTypeName The name of the Create Table As Select(CTAS) Type.
- * @param ctasInputTableAttribute The attribute in CTAS Type that associates it to the ''Input'' tables.
- * @param ctasOutputTableAttribute The attribute in CTAS Type that associates it to the ''Output'' tables.
- * @param depth depth as needed by the closure Query.
- * @param selectAttributes as needed by the closure Query.
- * @param withPath as needed by the closure Query.
- * @param persistenceStrategy as needed to evaluate the Closure Query.
- * @param g as needed to evaluate the Closure Query.
- */
-case class HiveWhereUsedQuery(tableTypeName : String,
- tableName : String,
- ctasTypeName : String,
- ctasInputTableAttribute : String,
- ctasOutputTableAttribute : String,
- depth : Option[Int],
- selectAttributes : Option[List[String]],
- withPath : Boolean,
- persistenceStrategy: GraphPersistenceStrategies,
- g: TitanGraph
- ) extends SingleInstanceClosureQuery[String] {
-
- val closureType : String = tableTypeName
-
- val attributeToSelectInstance = "name"
- val attributeTyp = DataTypes.STRING_TYPE
-
- val instanceValue = tableName
-
- lazy val closureRelation = List(
- ReverseRelation(ctasTypeName, ctasInputTableAttribute),
- Relation(ctasOutputTableAttribute)
- )
-}
-
-case class GraphResult(query: String, result : ITypedStruct) {
-
- def toTypedJson = Serialization.toJson(result)
-
- def toInstanceJson = InstanceSerialization.toJson(result)
-}
\ No newline at end of file
[15/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
new file mode 100755
index 0000000..6c3bd79
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedStruct;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class StructType extends AbstractDataType<IStruct>
+ implements IConstructableType<IStruct, ITypedStruct> {
+
+ public final TypeSystem typeSystem;
+ public final String name;
+ public final FieldMapping fieldMapping;
+ public final Map<AttributeInfo, List<String>> infoToNameMap;
+ public final int numFields;
+ private final TypedStructHandler handler;
+
+ /**
+ * Used when creating a StructType, to support recursive Structs.
+ */
+ protected StructType(TypeSystem typeSystem, String name, int numFields) {
+ this.typeSystem = typeSystem;
+ this.name = name;
+ this.fieldMapping = null;
+ infoToNameMap = null;
+ this.numFields = numFields;
+ this.handler = null;
+ }
+
+ protected StructType(TypeSystem typeSystem, String name,
+ ImmutableList<String> superTypes, AttributeInfo... fields)
+ throws MetadataException {
+ this.typeSystem = typeSystem;
+ this.name = name;
+ this.fieldMapping = constructFieldMapping(superTypes,
+ fields);
+ infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping);
+ this.numFields = this.fieldMapping.fields.size();
+ this.handler = new TypedStructHandler(this);
+ }
+
+ public FieldMapping fieldMapping() {
+ return fieldMapping;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ protected FieldMapping constructFieldMapping(ImmutableList<String> superTypes,
+ AttributeInfo... fields)
+ throws MetadataException {
+
+ Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
+ Map<String, Integer> fieldPos = new HashMap<String, Integer>();
+ Map<String, Integer> fieldNullPos = new HashMap<String, Integer>();
+ int numBools = 0;
+ int numBytes = 0;
+ int numShorts = 0;
+ int numInts = 0;
+ int numLongs = 0;
+ int numFloats = 0;
+ int numDoubles = 0;
+ int numBigInts = 0;
+ int numBigDecimals = 0;
+ int numDates = 0;
+ int numStrings = 0;
+ int numArrays = 0;
+ int numMaps = 0;
+ int numStructs = 0;
+ int numReferenceables = 0;
+
+ for (AttributeInfo i : fields) {
+ if (fieldsMap.containsKey(i.name)) {
+ throw new MetadataException(
+ String.format(
+ "Struct defintion cannot contain multiple fields with the same " +
+ "name %s",
+ i.name));
+ }
+ fieldsMap.put(i.name, i);
+ fieldNullPos.put(i.name, fieldNullPos.size());
+ if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
+ fieldPos.put(i.name, numBools);
+ numBools++;
+ } else if (i.dataType() == DataTypes.BYTE_TYPE) {
+ fieldPos.put(i.name, numBytes);
+ numBytes++;
+ } else if (i.dataType() == DataTypes.SHORT_TYPE) {
+ fieldPos.put(i.name, numShorts);
+ numShorts++;
+ } else if (i.dataType() == DataTypes.INT_TYPE) {
+ fieldPos.put(i.name, numInts);
+ numInts++;
+ } else if (i.dataType() == DataTypes.LONG_TYPE) {
+ fieldPos.put(i.name, numLongs);
+ numLongs++;
+ } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
+ fieldPos.put(i.name, numFloats);
+ numFloats++;
+ } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
+ fieldPos.put(i.name, numDoubles);
+ numDoubles++;
+ } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ fieldPos.put(i.name, numBigInts);
+ numBigInts++;
+ } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ fieldPos.put(i.name, numBigDecimals);
+ numBigDecimals++;
+ } else if (i.dataType() == DataTypes.DATE_TYPE) {
+ fieldPos.put(i.name, numDates);
+ numDates++;
+ } else if (i.dataType() == DataTypes.STRING_TYPE) {
+ fieldPos.put(i.name, numStrings);
+ numStrings++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
+ fieldPos.put(i.name, numInts);
+ numInts++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ fieldPos.put(i.name, numArrays);
+ numArrays++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ fieldPos.put(i.name, numMaps);
+ numMaps++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+ i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ fieldPos.put(i.name, numStructs);
+ numStructs++;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ fieldPos.put(i.name, numReferenceables);
+ numReferenceables++;
+ } else {
+ throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
+ }
+ }
+
+ return new FieldMapping(fieldsMap,
+ fieldPos,
+ fieldNullPos,
+ numBools,
+ numBytes,
+ numShorts,
+ numInts,
+ numLongs,
+ numFloats,
+ numDoubles,
+ numBigInts,
+ numBigDecimals,
+ numDates,
+ numStrings,
+ numArrays,
+ numMaps,
+ numStructs,
+ numReferenceables);
+ }
+
+
+ @Override
+ public DataTypes.TypeCategory getTypeCategory() {
+ return DataTypes.TypeCategory.STRUCT;
+ }
+
+ @Override
+ public ITypedStruct convert(Object val, Multiplicity m) throws MetadataException {
+ return handler.convert(val, m);
+ }
+
+ public ITypedStruct createInstance() {
+ return handler.createInstance();
+ }
+
+ @Override
+ public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
+ handler.output(s, buf, prefix);
+ }
+
+ public List<String> getNames(AttributeInfo info) {
+ return infoToNameMap.get(info);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
new file mode 100755
index 0000000..be6228e
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.ParamChecker;
+
+import java.util.Arrays;
+
+public class StructTypeDefinition {
+
+ public final String typeName;
+ public final AttributeDefinition[] attributeDefinitions;
+
+ protected StructTypeDefinition(String typeName, boolean validate, AttributeDefinition... attributeDefinitions) {
+ this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
+ if (attributeDefinitions != null && attributeDefinitions.length != 0) {
+ ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
+ }
+ this.attributeDefinitions = attributeDefinitions;
+ }
+
+ public StructTypeDefinition(String typeName, AttributeDefinition[] attributeDefinitions) {
+ this.typeName = ParamChecker.notEmpty(typeName, "Struct type name");
+ this.attributeDefinitions = ParamChecker.notNullElements(attributeDefinitions, "Attribute definitions");
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ StructTypeDefinition that = (StructTypeDefinition) o;
+
+ if (!Arrays.equals(attributeDefinitions, that.attributeDefinitions)) return false;
+ if (!typeName.equals(that.typeName)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = typeName.hashCode();
+ result = 31 * result + Arrays.hashCode(attributeDefinitions);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
new file mode 100755
index 0000000..06054d5
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedStruct;
+
+import java.util.List;
+import java.util.Map;
+
+public class TraitType extends HierarchicalType<TraitType, IStruct>
+ implements IConstructableType<IStruct, ITypedStruct> {
+
+ public final Map<AttributeInfo, List<String>> infoToNameMap;
+ private final TypedStructHandler handler;
+
+ /**
+ * Used when creating a TraitType, to support recursive Structs.
+ */
+ TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits,
+ int numFields) {
+ super(typeSystem, TraitType.class, name, superTraits, numFields);
+ handler = null;
+ infoToNameMap = null;
+ }
+
+ TraitType(TypeSystem typeSystem, String name, ImmutableList<String> superTraits,
+ AttributeInfo... fields)
+ throws MetadataException {
+ super(typeSystem, TraitType.class, name, superTraits, fields);
+ handler = new TypedStructHandler(this);
+ infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
+ }
+
+ @Override
+ public DataTypes.TypeCategory getTypeCategory() {
+ return DataTypes.TypeCategory.TRAIT;
+ }
+
+ @Override
+ public ITypedStruct convert(Object val, Multiplicity m) throws MetadataException {
+ return handler.convert(val, m);
+ }
+
+ public ITypedStruct createInstance() {
+ return handler.createInstance();
+ }
+
+ @Override
+ public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
+ handler.output(s, buf, prefix);
+ }
+
+ @Override
+ public List<String> getNames(AttributeInfo info) {
+ return infoToNameMap.get(info);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
new file mode 100755
index 0000000..3d7da00
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
@@ -0,0 +1,673 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Multimap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.classification.InterfaceAudience;
+import org.apache.atlas.typesystem.TypesDef;
+
+import javax.inject.Singleton;
+import java.lang.reflect.Constructor;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+
+@Singleton
+@InterfaceAudience.Private
+public class TypeSystem {
+ private static final TypeSystem INSTANCE = new TypeSystem();
+ private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() {
+ @Override
+ public SimpleDateFormat initialValue() {
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return dateFormat;
+ }
+ };
+
+ private Map<String, IDataType> types;
+ private IdType idType;
+
+ /**
+ * An in-memory copy of type categories vs types for convenience.
+ */
+ private Multimap<DataTypes.TypeCategory, String> typeCategoriesToTypeNamesMap;
+
+ private ImmutableList<String> coreTypes;
+
+ public TypeSystem() {
+ initialize();
+ }
+
+ public static TypeSystem getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * This is only used for testing purposes. Not intended for public use.
+ */
+ @InterfaceAudience.Private
+ public void reset() {
+ initialize();
+ }
+
+ private void initialize() {
+ types = new ConcurrentHashMap<>();
+ typeCategoriesToTypeNamesMap = ArrayListMultimap.create(DataTypes.TypeCategory.values().length, 10);
+
+ registerPrimitiveTypes();
+ registerCoreTypes();
+ coreTypes = ImmutableList.copyOf(types.keySet());
+ }
+
+ public ImmutableList<String> getCoreTypes() {
+ return coreTypes;
+ }
+
+ public ImmutableList<String> getTypeNames() {
+ return ImmutableList.copyOf(types.keySet());
+ }
+
+ public ImmutableList<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) {
+ return ImmutableList.copyOf(typeCategoriesToTypeNamesMap.get(typeCategory));
+ }
+
+ private void registerPrimitiveTypes() {
+ types.put(DataTypes.BOOLEAN_TYPE.getName(), DataTypes.BOOLEAN_TYPE);
+ types.put(DataTypes.BYTE_TYPE.getName(), DataTypes.BYTE_TYPE);
+ types.put(DataTypes.SHORT_TYPE.getName(), DataTypes.SHORT_TYPE);
+ types.put(DataTypes.INT_TYPE.getName(), DataTypes.INT_TYPE);
+ types.put(DataTypes.LONG_TYPE.getName(), DataTypes.LONG_TYPE);
+ types.put(DataTypes.FLOAT_TYPE.getName(), DataTypes.FLOAT_TYPE);
+ types.put(DataTypes.DOUBLE_TYPE.getName(), DataTypes.DOUBLE_TYPE);
+ types.put(DataTypes.BIGINTEGER_TYPE.getName(), DataTypes.BIGINTEGER_TYPE);
+ types.put(DataTypes.BIGDECIMAL_TYPE.getName(), DataTypes.BIGDECIMAL_TYPE);
+ types.put(DataTypes.DATE_TYPE.getName(), DataTypes.DATE_TYPE);
+ types.put(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE);
+
+ typeCategoriesToTypeNamesMap.putAll(DataTypes.TypeCategory.PRIMITIVE, types.keySet());
+ }
+
+
+ /*
+ * The only core OOB type we will define is the Struct to represent the Identity of an Instance.
+ */
+ private void registerCoreTypes() {
+ idType = new IdType();
+ }
+
+ public IdType getIdType() {
+ return idType;
+ }
+
+ public boolean isRegistered(String typeName) {
+ return types.containsKey(typeName);
+ }
+
+ public <T> T getDataType(Class<T> cls, String name) throws MetadataException {
+ if (types.containsKey(name)) {
+ try {
+ return cls.cast(types.get(name));
+ } catch(ClassCastException cce) {
+ throw new MetadataException(cce);
+ }
+ }
+
+ /*
+ * is this an Array Type?
+ */
+ String arrElemType = TypeUtils.parseAsArrayType(name);
+ if (arrElemType != null) {
+ IDataType dT = defineArrayType(getDataType(IDataType.class, arrElemType));
+ return cls.cast(dT);
+ }
+
+ /*
+ * is this a Map Type?
+ */
+ String[] mapType = TypeUtils.parseAsMapType(name);
+ if (mapType != null) {
+ IDataType dT = defineMapType(getDataType(IDataType.class, mapType[0]),
+ getDataType(IDataType.class, mapType[1]));
+ return cls.cast(dT);
+ }
+
+ throw new MetadataException(String.format("Unknown datatype: %s", name));
+ }
+
+ public StructType defineStructType(String name,
+ boolean errorIfExists,
+ AttributeDefinition... attrDefs) throws MetadataException {
+ StructTypeDefinition structDef = new StructTypeDefinition(name, attrDefs);
+ defineTypes(ImmutableList.of(structDef),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+
+ return getDataType(StructType.class, structDef.typeName);
+ }
+
+ /**
+ * construct a temporary StructType for a Query Result. This is not registered in the
+ * typeSystem.
+ * The attributes in the typeDefinition can only reference permanent types.
+ * @param name struct type name
+ * @param attrDefs struct type definition
+ * @return temporary struct type
+ * @throws MetadataException
+ */
+ public StructType defineQueryResultType(String name,
+ Map<String, IDataType> tempTypes,
+ AttributeDefinition... attrDefs)
+ throws MetadataException {
+
+ AttributeInfo[] infos = new AttributeInfo[attrDefs.length];
+ for (int i = 0; i < attrDefs.length; i++) {
+ infos[i] = new AttributeInfo(this, attrDefs[i], tempTypes);
+ }
+
+ return new StructType(this, name, null, infos);
+ }
+
+ public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef)
+ throws MetadataException {
+
+ defineTypes(ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(traitDef),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+
+ return getDataType(TraitType.class, traitDef.typeName);
+ }
+
+ public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef)
+ throws MetadataException {
+
+ defineTypes(ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(classDef));
+
+ return getDataType(ClassType.class, classDef.typeName);
+ }
+
+ public Map<String, IDataType> defineTraitTypes(
+ HierarchicalTypeDefinition<TraitType>... traitDefs) throws MetadataException {
+ TransientTypeSystem transientTypes = new TransientTypeSystem(
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.copyOf(traitDefs),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+ return transientTypes.defineTypes();
+ }
+
+ public Map<String, IDataType> defineClassTypes(
+ HierarchicalTypeDefinition<ClassType>... classDefs) throws MetadataException {
+ TransientTypeSystem transientTypes = new TransientTypeSystem(
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.copyOf(classDefs));
+ return transientTypes.defineTypes();
+ }
+
+ public Map<String, IDataType> defineTypes(TypesDef typesDef)
+ throws MetadataException {
+ Map<String, IDataType> typesAdded = new HashMap<>();
+ for (EnumTypeDefinition enumDef : typesDef.enumTypesAsJavaList()) {
+ typesAdded.put(enumDef.name, defineEnumType(enumDef));
+ }
+
+ ImmutableList<StructTypeDefinition> structDefs = ImmutableList
+ .copyOf(typesDef.structTypesAsJavaList());
+ ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs =
+ ImmutableList.copyOf(typesDef.traitTypesAsJavaList());
+ ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs =
+ ImmutableList.copyOf(typesDef.classTypesAsJavaList());
+
+ typesAdded.putAll(defineTypes(structDefs, traitDefs, classDefs));
+ return typesAdded;
+ }
+
+ public Map<String, IDataType> defineTypes(ImmutableList<StructTypeDefinition> structDefs,
+ ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
+ ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs)
+ throws MetadataException {
+ TransientTypeSystem transientTypes = new TransientTypeSystem(structDefs,
+ traitDefs,
+ classDefs);
+ Map<String, IDataType> definedTypes = transientTypes.defineTypes();
+// LOG.debug("Defined new types " + Arrays.toString(definedTypes.keySet().toArray(new String[definedTypes.size()])));
+ return definedTypes;
+ }
+
+ public DataTypes.ArrayType defineArrayType(IDataType elemType) throws MetadataException {
+ assert elemType != null;
+ DataTypes.ArrayType dT = new DataTypes.ArrayType(elemType);
+ types.put(dT.getName(), dT);
+ typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ARRAY, dT.getName());
+ return dT;
+ }
+
+ public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType)
+ throws MetadataException {
+ assert keyType != null;
+ assert valueType != null;
+ DataTypes.MapType dT = new DataTypes.MapType(keyType, valueType);
+ types.put(dT.getName(), dT);
+ typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.MAP, dT.getName());
+ return dT;
+ }
+
+ public EnumType defineEnumType(String name, EnumValue... values) throws MetadataException {
+ return defineEnumType(new EnumTypeDefinition(name, values));
+ }
+
+ public EnumType defineEnumType(EnumTypeDefinition eDef) throws MetadataException {
+ assert eDef.name != null;
+ if (types.containsKey(eDef.name)) {
+ throw new MetadataException(
+ String.format("Redefinition of type %s not supported", eDef.name));
+ }
+
+ EnumType eT = new EnumType(this, eDef.name, eDef.enumValues);
+ types.put(eDef.name, eT);
+ typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ENUM, eDef.name);
+ return eT;
+ }
+
+ public SimpleDateFormat getDateFormat() {
+ return dateFormat.get();
+ }
+
+ public boolean allowNullsInCollections() {
+ return false;
+ }
+
+ public void removeTypes(ImmutableList<String> typeNames) {
+
+ }
+
+ class TransientTypeSystem extends TypeSystem {
+
+ final ImmutableList<StructTypeDefinition> structDefs;
+ final ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs;
+ final ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs;
+ Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<>();
+ Map<String, HierarchicalTypeDefinition<TraitType>> traitNameToDefMap =
+ new HashMap<>();
+ Map<String, HierarchicalTypeDefinition<ClassType>> classNameToDefMap =
+ new HashMap<>();
+
+ Set<String> transientTypes;
+
+ List<AttributeInfo> recursiveRefs;
+ List<DataTypes.ArrayType> recursiveArrayTypes;
+ List<DataTypes.MapType> recursiveMapTypes;
+
+
+ TransientTypeSystem(ImmutableList<StructTypeDefinition> structDefs,
+ ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
+ ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) {
+
+ this.structDefs = structDefs;
+ this.traitDefs = traitDefs;
+ this.classDefs = classDefs;
+ structNameToDefMap = new HashMap<>();
+ traitNameToDefMap = new HashMap<>();
+ classNameToDefMap = new HashMap<>();
+
+ recursiveRefs = new ArrayList<>();
+ recursiveArrayTypes = new ArrayList<>();
+ recursiveMapTypes = new ArrayList<>();
+ transientTypes = new LinkedHashSet<>();
+ }
+
+ private IDataType dataType(String name) {
+ return TypeSystem.this.types.get(name);
+ }
+
+ /*
+ * Step 1:
+ * - validate cannot redefine types
+ * - setup shallow Type instances to facilitate recursive type graphs
+ */
+ private void step1() throws MetadataException {
+ for (StructTypeDefinition sDef : structDefs) {
+ assert sDef.typeName != null;
+ if (dataType(sDef.typeName) != null) {
+ throw new MetadataException(
+ String.format("Cannot redefine type %s", sDef.typeName));
+ }
+ TypeSystem.this.types.put(sDef.typeName,
+ new StructType(this, sDef.typeName, sDef.attributeDefinitions.length));
+ structNameToDefMap.put(sDef.typeName, sDef);
+ transientTypes.add(sDef.typeName);
+ }
+
+ for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) {
+ assert traitDef.typeName != null;
+ if (types.containsKey(traitDef.typeName)) {
+ throw new MetadataException(
+ String.format("Cannot redefine type %s", traitDef.typeName));
+ }
+
+ TypeSystem.this.types.put(traitDef.typeName,
+ new TraitType(this, traitDef.typeName, traitDef.superTypes,
+ traitDef.attributeDefinitions.length));
+ traitNameToDefMap.put(traitDef.typeName, traitDef);
+ transientTypes.add(traitDef.typeName);
+ }
+
+ for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) {
+ assert classDef.typeName != null;
+ if (types.containsKey(classDef.typeName)) {
+ throw new MetadataException(
+ String.format("Cannot redefine type %s", classDef.typeName));
+ }
+
+ TypeSystem.this.types.put(classDef.typeName,
+ new ClassType(this, classDef.typeName, classDef.superTypes,
+ classDef.attributeDefinitions.length));
+ classNameToDefMap.put(classDef.typeName, classDef);
+ transientTypes.add(classDef.typeName);
+ }
+ }
+
+ private <U extends HierarchicalType> void validateSuperTypes(Class<U> cls,
+ HierarchicalTypeDefinition<U> def)
+ throws MetadataException {
+ Set<String> s = new HashSet<>();
+ ImmutableList<String> superTypes = def.superTypes;
+ for (String superTypeName : superTypes) {
+
+ if (s.contains(superTypeName)) {
+ throw new MetadataException(
+ String.format("Type %s extends superType %s multiple times",
+ def.typeName, superTypeName));
+ }
+
+ IDataType dT = dataType(superTypeName);
+
+ if (dT == null) {
+ throw new MetadataException(
+ String.format("Unknown superType %s in definition of type %s",
+ superTypeName, def.typeName));
+ }
+
+ if (!cls.isAssignableFrom(dT.getClass())) {
+ throw new MetadataException(
+ String.format("SuperType %s must be a %s, in definition of type %s",
+ superTypeName, cls.getName(), def.typeName));
+ }
+ s.add(superTypeName);
+ }
+ }
+
+ /*
+ * Step 2:
+ * - for Hierarchical Types, validate SuperTypes.
+ * - for each Hierarchical Type setup their SuperTypes Graph
+ */
+ private void step2() throws MetadataException {
+ for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) {
+ validateSuperTypes(TraitType.class, traitDef);
+ TraitType traitType = getDataType(TraitType.class, traitDef.typeName);
+ traitType.setupSuperTypesGraph();
+ }
+
+ for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) {
+ validateSuperTypes(ClassType.class, classDef);
+ ClassType classType = getDataType(ClassType.class, classDef.typeName);
+ classType.setupSuperTypesGraph();
+ }
+ }
+
+ private AttributeInfo constructAttributeInfo(AttributeDefinition attrDef)
+ throws MetadataException {
+ AttributeInfo info = new AttributeInfo(this, attrDef, null);
+ if (transientTypes.contains(attrDef.dataTypeName)) {
+ recursiveRefs.add(info);
+ }
+ if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ DataTypes.ArrayType arrType = (DataTypes.ArrayType) info.dataType();
+ if (transientTypes.contains(arrType.getElemType().getName())) {
+ recursiveArrayTypes.add(arrType);
+ }
+ }
+ if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ DataTypes.MapType mapType = (DataTypes.MapType) info.dataType();
+ if (transientTypes.contains(mapType.getKeyType().getName())) {
+ recursiveMapTypes.add(mapType);
+ } else if (transientTypes.contains(mapType.getValueType().getName())) {
+ recursiveMapTypes.add(mapType);
+ }
+ }
+
+ if (info.multiplicity.upper > 1 && !(
+ info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP ||
+ info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY)) {
+ throw new MetadataException(
+ String.format("A multiplicty of more than one requires a collection type for attribute '%s'",
+ info.name));
+ }
+
+ return info;
+ }
+
+ private StructType constructStructureType(StructTypeDefinition def)
+ throws MetadataException {
+ AttributeInfo[] infos = new AttributeInfo[def.attributeDefinitions.length];
+ for (int i = 0; i < def.attributeDefinitions.length; i++) {
+ infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
+ }
+
+ StructType type = new StructType(TypeSystem.this, def.typeName, null, infos);
+ TypeSystem.this.types.put(def.typeName, type);
+ return type;
+ }
+
+ private <U extends HierarchicalType> U constructHierarchicalType(Class<U> cls,
+ HierarchicalTypeDefinition<U> def)
+ throws MetadataException {
+ AttributeInfo[] infos = new AttributeInfo[def.attributeDefinitions.length];
+ for (int i = 0; i < def.attributeDefinitions.length; i++) {
+ infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
+ }
+
+ try {
+ Constructor<U> cons = cls.getDeclaredConstructor(
+ TypeSystem.class,
+ String.class,
+ ImmutableList.class,
+ AttributeInfo[].class);
+ U type = cons.newInstance(TypeSystem.this, def.typeName, def.superTypes, infos);
+ TypeSystem.this.types.put(def.typeName, type);
+ return type;
+ } catch (Exception e) {
+ throw new MetadataException(
+ String.format("Cannot construct Type of MetaType %s", cls.getName()), e);
+ }
+ }
+
+ /*
+ * Step 3:
+ * - Order Hierarchical Types in order of SuperType before SubType.
+ * - Construct all the Types
+ */
+ private void step3() throws MetadataException {
+
+ List<TraitType> traitTypes = new ArrayList<>();
+ for (String traitTypeName : traitNameToDefMap.keySet()) {
+ traitTypes.add(getDataType(TraitType.class, traitTypeName));
+ }
+ Collections.sort(traitTypes);
+
+ List<ClassType> classTypes = new ArrayList<>();
+ for (String classTypeName : classNameToDefMap.keySet()) {
+ classTypes.add(getDataType(ClassType.class, classTypeName));
+ }
+ Collections.sort(classTypes);
+
+ for (StructTypeDefinition structDef : structDefs) {
+ constructStructureType(structDef);
+ typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.CLASS, structDef.typeName);
+ }
+
+ for (TraitType traitType : traitTypes) {
+ constructHierarchicalType(TraitType.class,
+ traitNameToDefMap.get(traitType.getName()));
+ typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.TRAIT, traitType.getName());
+ }
+
+ for (ClassType classType : classTypes) {
+ constructHierarchicalType(ClassType.class,
+ classNameToDefMap.get(classType.getName()));
+ typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.CLASS, classType.getName());
+ }
+ }
+
+ /*
+ * Step 4:
+ * - fix up references in recursive AttrInfo and recursive Collection Types.
+ */
+ private void step4() throws MetadataException {
+ for (AttributeInfo info : recursiveRefs) {
+ info.setDataType(dataType(info.dataType().getName()));
+ }
+ for (DataTypes.ArrayType arrType : recursiveArrayTypes) {
+ arrType.setElemType(dataType(arrType.getElemType().getName()));
+ }
+ for (DataTypes.MapType mapType : recursiveMapTypes) {
+ mapType.setKeyType(dataType(mapType.getKeyType().getName()));
+ mapType.setValueType(dataType(mapType.getValueType().getName()));
+ }
+ }
+
+ Map<String, IDataType> defineTypes() throws MetadataException {
+ step1();
+ step2();
+ try {
+ step3();
+ step4();
+ } catch (MetadataException me) {
+ for (String sT : transientTypes) {
+ types.remove(sT);
+ }
+ throw me;
+ }
+
+ Map<String, IDataType> newTypes = new HashMap<>();
+
+ for (String tName : transientTypes) {
+ newTypes.put(tName, dataType(tName));
+ }
+ return newTypes;
+ }
+
+ @Override
+ public ImmutableList<String> getTypeNames() {
+ return TypeSystem.this.getTypeNames();
+ }
+
+ @Override
+ public <T> T getDataType(Class<T> cls, String name) throws MetadataException {
+ return TypeSystem.this.getDataType(cls, name);
+ }
+
+ @Override
+ public StructType defineStructType(String name, boolean errorIfExists,
+ AttributeDefinition... attrDefs)
+ throws MetadataException {
+ throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
+ }
+
+ @Override
+ public TraitType defineTraitType(HierarchicalTypeDefinition traitDef)
+ throws MetadataException {
+ throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
+ }
+
+ @Override
+ public ClassType defineClassType(HierarchicalTypeDefinition<ClassType> classDef
+ ) throws MetadataException {
+ throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
+ }
+
+ @Override
+ public Map<String, IDataType> defineTypes(ImmutableList<StructTypeDefinition> structDefs,
+ ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
+ ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs)
+ throws MetadataException {
+ throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
+ }
+
+ @Override
+ public DataTypes.ArrayType defineArrayType(IDataType elemType) throws MetadataException {
+ throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
+ }
+
+ @Override
+ public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType)
+ throws MetadataException {
+ throw new MetadataException("Internal Error: define type called on TrasientTypeSystem");
+ }
+ }
+
+ public class IdType {
+ private static final String ID_ATTRNAME = "guid";
+ private static final String TYPENAME_ATTRNAME = "typeName";
+ private static final String TYP_NAME = "__IdType";
+
+ private IdType() {
+ AttributeDefinition idAttr = new AttributeDefinition(ID_ATTRNAME,
+ DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null);
+ AttributeDefinition typNmAttr =
+ new AttributeDefinition(TYPENAME_ATTRNAME,
+ DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null);
+ try {
+ AttributeInfo[] infos = new AttributeInfo[2];
+ infos[0] = new AttributeInfo(TypeSystem.this, idAttr, null);
+ infos[1] = new AttributeInfo(TypeSystem.this, typNmAttr, null);
+
+ StructType type = new StructType(TypeSystem.this, TYP_NAME, null, infos);
+ TypeSystem.this.types.put(TYP_NAME, type);
+
+ } catch (MetadataException me) {
+ throw new RuntimeException(me);
+ }
+ }
+
+ public StructType getStructType() throws MetadataException {
+ return getDataType(StructType.class, TYP_NAME);
+ }
+
+ public String getName() { return TYP_NAME; }
+ public String idAttrName() { return ID_ATTRNAME;}
+ public String typeNameAttrName() { return TYPENAME_ATTRNAME;}
+ }
+
+ public static final String ID_STRUCT_ID_ATTRNAME = IdType.ID_ATTRNAME;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
new file mode 100755
index 0000000..0dbbf5e
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.TypesDef;
+import scala.collection.JavaConversions;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class TypeUtils {
+
+ public static final String NAME_REGEX = "[a-zA-z][a-zA-Z0-9_]*";
+ public static final Pattern NAME_PATTERN = Pattern.compile(NAME_REGEX);
+ public static final Pattern ARRAY_TYPE_NAME_PATTERN = Pattern
+ .compile(String.format("array<(%s)>", NAME_REGEX));
+ public static final Pattern MAP_TYPE_NAME_PATTERN =
+ Pattern.compile(String.format("map<(%s),(%s)>", NAME_REGEX, NAME_REGEX));
+
+ public static void outputVal(String val, Appendable buf, String prefix)
+ throws MetadataException {
+ try {
+ buf.append(prefix).append(val);
+ } catch (IOException ie) {
+ throw new MetadataException(ie);
+ }
+ }
+
+ public static String parseAsArrayType(String typeName) {
+ Matcher m = ARRAY_TYPE_NAME_PATTERN.matcher(typeName);
+ return m.matches() ? m.group(1) : null;
+ }
+
+ public static String[] parseAsMapType(String typeName) {
+ Matcher m = MAP_TYPE_NAME_PATTERN.matcher(typeName);
+ return m.matches() ? new String[]{m.group(1), m.group(2)} : null;
+ }
+
+ public static Map<AttributeInfo, List<String>> buildAttrInfoToNameMap(FieldMapping f) {
+ Map<AttributeInfo, List<String>> b = new HashMap<AttributeInfo, List<String>>();
+ for (Map.Entry<String, AttributeInfo> e : f.fields.entrySet()) {
+ List<String> names = b.get(e.getValue());
+ if (names == null) {
+ names = new ArrayList<String>();
+ b.put(e.getValue(), names);
+ }
+ names.add(e.getKey());
+ }
+ return ImmutableMap.copyOf(b);
+ }
+
+ public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums, ImmutableList<StructTypeDefinition> structs,
+ ImmutableList<HierarchicalTypeDefinition<TraitType>> traits,
+ ImmutableList<HierarchicalTypeDefinition<ClassType>> classes) {
+ return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
+ JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
+ }
+
+ protected static class Pair<L,R> {
+ protected L left;
+ protected R right;
+
+ public Pair(L left, R right) {
+ this.left = left;
+ this.right = right;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java
new file mode 100755
index 0000000..2ca9c85
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypedStructHandler.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.StructInstance;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.Map;
+
+public class TypedStructHandler {
+
+ private final IConstructableType<IStruct, ITypedStruct> structType;
+ private final FieldMapping fieldMapping;
+
+ public TypedStructHandler(IConstructableType<IStruct, ITypedStruct> structType) {
+ this.structType = structType;
+ fieldMapping = structType.fieldMapping();
+ }
+
+ public ITypedStruct convert(Object val, Multiplicity m) throws MetadataException {
+ if (val != null) {
+ if (val instanceof ITypedStruct) {
+ ITypedStruct ts = (ITypedStruct) val;
+ if (ts.getTypeName() != structType.getName()) {
+ throw new ValueConversionException(structType, val);
+ }
+ return ts;
+ } else if (val instanceof Struct) {
+ Struct s = (Struct) val;
+ if (!s.typeName.equals(structType.getName())) {
+ throw new ValueConversionException(structType, val);
+ }
+ ITypedStruct ts = createInstance();
+ for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
+ String attrKey = e.getKey();
+ AttributeInfo i = e.getValue();
+ Object aVal = s.get(attrKey);
+ try {
+ ts.set(attrKey, aVal);
+ } catch (ValueConversionException ve) {
+ throw new ValueConversionException(structType, val, ve);
+ }
+ }
+ return ts;
+ } else if (val instanceof StructInstance &&
+ ((StructInstance) val).getTypeName() == structType.getName()) {
+ return (StructInstance) val;
+ } else {
+ throw new ValueConversionException(structType, val);
+ }
+ }
+ if (!m.nullAllowed()) {
+ throw new ValueConversionException.NullConversionException(m);
+ }
+ return null;
+ }
+
+ public DataTypes.TypeCategory getTypeCategory() {
+ return DataTypes.TypeCategory.STRUCT;
+ }
+
+ public ITypedStruct createInstance() {
+ return new StructInstance(structType.getName(),
+ fieldMapping,
+ new boolean[fieldMapping.fields.size()],
+ fieldMapping.numBools == 0 ? null : new boolean[fieldMapping.numBools],
+ fieldMapping.numBytes == 0 ? null : new byte[fieldMapping.numBytes],
+ fieldMapping.numShorts == 0 ? null : new short[fieldMapping.numShorts],
+ fieldMapping.numInts == 0 ? null : new int[fieldMapping.numInts],
+ fieldMapping.numLongs == 0 ? null : new long[fieldMapping.numLongs],
+ fieldMapping.numFloats == 0 ? null : new float[fieldMapping.numFloats],
+ fieldMapping.numDoubles == 0 ? null : new double[fieldMapping.numDoubles],
+ fieldMapping.numBigDecimals == 0 ? null
+ : new BigDecimal[fieldMapping.numBigDecimals],
+ fieldMapping.numBigInts == 0 ? null : new BigInteger[fieldMapping.numBigInts],
+ fieldMapping.numDates == 0 ? null : new Date[fieldMapping.numDates],
+ fieldMapping.numStrings == 0 ? null : new String[fieldMapping.numStrings],
+ fieldMapping.numArrays == 0 ? null : new ImmutableList[fieldMapping.numArrays],
+ fieldMapping.numMaps == 0 ? null : new ImmutableMap[fieldMapping.numMaps],
+ fieldMapping.numStructs == 0 ? null : new StructInstance[fieldMapping.numStructs],
+ fieldMapping.numReferenceables == 0 ? null
+ : new ReferenceableInstance[fieldMapping.numReferenceables],
+ fieldMapping.numReferenceables == 0 ? null
+ : new Id[fieldMapping.numReferenceables]);
+ }
+
+ public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
+ fieldMapping.output(s, buf, prefix);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java
new file mode 100755
index 0000000..dfeb679
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ValueConversionException.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.MetadataException;
+
+public class ValueConversionException extends MetadataException {
+
+ public ValueConversionException(IDataType typ, Object val) {
+ this(typ, val, (Throwable) null);
+ }
+
+ public ValueConversionException(IDataType typ, Object val, Throwable t) {
+ super(String
+ .format("Cannot convert value '%s' to datatype %s", val.toString(), typ.getName()),
+ t);
+ }
+
+ public ValueConversionException(IDataType typ, Object val, String msg) {
+ super(String.format("Cannot convert value '%s' to datatype %s because: %s",
+ val.toString(), typ.getName(), msg));
+ }
+
+ public ValueConversionException(String typeName, Object val, String msg) {
+ super(String.format("Cannot convert value '%s' to datatype %s because: %s",
+ val.toString(), typeName, msg));
+ }
+
+ protected ValueConversionException(String msg) {
+ super(msg);
+ }
+
+ public static class NullConversionException extends ValueConversionException {
+ public NullConversionException(Multiplicity m) {
+ super(String.format("Null value not allowed for multiplicty %s", m));
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/package-info.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/package-info.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/package-info.java
new file mode 100755
index 0000000..fbd4216
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/package-info.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <h2>Types:</h2>
+ * <img src="doc-files/dataTypes.png" />
+ * <ul>
+ * <li> <b>IDataType:</b> Represents a <i>DataType</i> in the TypeSystem. All Instances and
+ * Attributes are associated
+ * with a DataType. They represent the <b>Set</b> of values that Instances/Attributes of this
+ * type can have.
+ * Currently the namespace of DataTypes is flat. DataTypes can be asked to <i>convert</i>
+ * arbitrary java Objects
+ * to instances of this type, and they can be asked for a String representation of an
+ * instance.</li>
+ * <li><b>Type Categories:</b></li> DataTypes are grouped into Categories. A Category implies
+ * certain semantics about
+ * the Types belonging to the Category. We have PRIMITIVE, ENUM, ARRAY, MAP, STRUCT, TRAIT,
+ * and CLASS categories.
+ * <li><b>Primitive Types:</b> There are corresponding DataTypes for the java primitives:
+ * Boolean, Byte, Short,
+ * Int, Long, Float, Double. We also support BigInteger, BigDecimal, String, and Date</li>
+ * <li><b>Collection Types:</b>ArrayType and MapType are parameterized DataTypes taking one
+ * and two parameters
+ * respectively.</li>
+ * <li><b>Enum Types:</b> Used to define DataTypes with all valid values listed in the Type
+ * definition. For e.g.
+ * <pre>
+ * {@code
+ * ts.defineEnumType("HiveObjectType",
+new EnumValue("GLOBAL", 1),
+new EnumValue("DATABASE", 2),
+new EnumValue("TABLE", 3),
+new EnumValue("PARTITION", 4),
+new EnumValue("COLUMN", 5))
+ * }
+ * </pre> Each <i>EnumValue</i> has name and an ordinal. Either one can be used as a value for an
+ * Attribute of this Type.
+ * </li>
+ * <li><b>Constructable Types:</b> Are complex Types that are composed of Attributes. We
+ * support Structs, Classes
+ * and Traits constructable types. A ConstructableType is parameterized by the Type of its
+ * <i>Instance</i> java
+ * class(these are implementations of the ITypedInstance interface). A value of the
+ * IConstructableType will
+ * implement this parameterized Type. IConstructableTypes can be asked to create an 'empty'
+ * instance of their Type.
+ * IConstructableTypes are associated with FieldMappings that encapsulate the mapping from/to
+ * the ITypedInstance
+ * java object.
+ * </li>
+ * <li><b>Attribute Info:</b>Represents an Attribute of a complex datatype. Attributes are
+ * defined by a name, a
+ * dataType, its Multiplicity and whether it is a composite relation. <i>Multiplicity</i> is
+ * a constraint on the
+ * number of instances that an instance can have. For non collection types and Maps:
+ * Multiplicity is OPTIONAL or
+ * REQUIRED.
+ * For Arrays the Multiplicity is specified by a lower-bound, upper-bound and a uniqueness
+ * constraint.
+ * </li>
+ * <li><b>Struct Types:</b>Are IConstructableTypes whose instances are IStructs. Conceptually
+ * these are like 'C'
+ * structs: they represent a collection of Attributes. For e.g.
+ * <pre>
+ * {@code
+ * ts.defineStructType(STRUCT_TYPE_1,
+true,
+createRequiredAttrDef("a", DataTypes.INT_TYPE),
+createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+createOptionalAttrDef("e", DataTypes.INT_TYPE),
+createOptionalAttrDef("f", DataTypes.INT_TYPE),
+createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE))
+ * }
+ * </pre>
+ * </li>
+ * <li><b>Hierarchical Types:</b>Are DataTypes that can have a SuperType. Classes and Traits
+ * are the supported
+ * Hierarchical Types. </li>
+ * <li><b>Class Types:</b></li>
+ * <li><b>Trait Types:</b></li>
+ * </ul>
+ *
+ *
+ * <h2>Instances:</h2>
+ * <img src="doc-files/instance.png" />
+ * <ul>
+ * <li> <b>IStruct:</b></li>
+ * <li><b>IReferenceableInstance:</b></li>
+ * <li><b>ITypedStruct:</b></li>
+ * <li><b>ITypedReferenceableInstance:</b></li>
+ * </ul>
+ *
+ * <h3>Serialization of Types:</h3>
+ *
+ * <h3>Serialization of Instances:</h3>
+ *
+ * <h3>Searching on Classes and Traits:</h3>
+ */
+package org.apache.atlas.typesystem.types;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
new file mode 100755
index 0000000..95f8cab
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types.utils;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+
+/**
+ * Types utilities class.
+ */
+public class TypesUtil {
+
+ private TypesUtil() {
+ }
+
+ public static AttributeDefinition createOptionalAttrDef(String name,
+ IDataType dataType) {
+ return new AttributeDefinition(name, dataType.getName(),
+ Multiplicity.OPTIONAL, false, null);
+ }
+
+ public static AttributeDefinition createOptionalAttrDef(String name,
+ String dataType) {
+ return new AttributeDefinition(name, dataType, Multiplicity.OPTIONAL, false, null);
+ }
+
+ public static AttributeDefinition createRequiredAttrDef(String name,
+ String dataType) {
+ return new AttributeDefinition(name, dataType, Multiplicity.REQUIRED, false, null);
+ }
+
+ public static AttributeDefinition createUniqueRequiredAttrDef(String name,
+ IDataType dataType) {
+ return new AttributeDefinition(name, dataType.getName(),
+ Multiplicity.REQUIRED, false, true, true, null);
+ }
+
+ public static AttributeDefinition createRequiredAttrDef(String name,
+ IDataType dataType) {
+ return new AttributeDefinition(name, dataType.getName(),
+ Multiplicity.REQUIRED, false, null);
+ }
+
+ public static EnumTypeDefinition createEnumTypeDef(String name, EnumValue... enumValues) {
+ return new EnumTypeDefinition(name, enumValues);
+ }
+
+ public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(
+ String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
+ return new HierarchicalTypeDefinition<>(TraitType.class, name, superTypes, attrDefs);
+ }
+
+ public static StructTypeDefinition createStructTypeDef(String name, AttributeDefinition... attrDefs) {
+ return new StructTypeDefinition(name, attrDefs);
+ }
+
+ public static HierarchicalTypeDefinition<ClassType> createClassTypeDef(
+ String name, ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
+ return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/MetadataException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/MetadataException.java b/typesystem/src/main/java/org/apache/hadoop/metadata/MetadataException.java
deleted file mode 100755
index 4a5aeca..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/MetadataException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-/**
- * Base Exception class for metadata API.
- */
-public class MetadataException extends Exception {
-
- public MetadataException() {
- }
-
- public MetadataException(String message) {
- super(message);
- }
-
- public MetadataException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public MetadataException(Throwable cause) {
- super(cause);
- }
-
- public MetadataException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/ParamChecker.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/ParamChecker.java b/typesystem/src/main/java/org/apache/hadoop/metadata/ParamChecker.java
deleted file mode 100644
index 58bb697..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/ParamChecker.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-public class ParamChecker {
-
- /**
- * Check that a value is not null. If null throws an IllegalArgumentException.
- *
- * @param obj value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static <T> T notNull(T obj, String name) {
- if (obj == null) {
- throw new IllegalArgumentException(name + " cannot be null");
- }
- return obj;
- }
-
- /**
- * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
- * throws an IllegalArgumentException.
- * @param list the list of T.
- * @param name parameter name for the exception message.
- */
- public static <T> Collection<T> notNullElements(Collection<T> list, String name) {
- notEmpty(list, name);
- for (T ele : list) {
- notNull(ele, String.format("Collection %s element %s", name, ele));
- }
- return list;
- }
-
- /**
- * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
- * throws an IllegalArgumentException.
- * @param array the array of T.
- * @param name parameter name for the exception message.
- */
- public static <T> T[] notNullElements(T[] array, String name) {
- notEmpty(Arrays.asList(array), name);
- for (T ele : array) {
- notNull(ele, String.format("Collection %s element %s", name, ele));
- }
- return array;
- }
-
- /**
- * Check that a list is not null and not empty.
- * @param list the list of T.
- * @param name parameter name for the exception message.
- */
- public static <T> Collection<T> notEmpty(Collection<T> list, String name) {
- notNull(list, name);
- if (list.isEmpty()) {
- throw new IllegalArgumentException(String.format("Collection %s is empty", name));
- }
- return list;
- }
-
- /**
- * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static String notEmpty(String value, String name) {
- return notEmpty(value, name, null);
- }
-
- /**
- * Check that a string is not empty if its not null.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static String notEmptyIfNotNull(String value, String name) {
- return notEmptyIfNotNull(value, name, null);
- }
-
- /**
- * Check that a string is not empty if its not null.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static String notEmptyIfNotNull(String value, String name, String info) {
- if (value == null) {
- return value;
- }
-
- if (value.trim().length() == 0) {
- throw new IllegalArgumentException(name + " cannot be empty" + (info == null ? "" : ", " + info));
- }
- return value.trim();
- }
-
- /**
- * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @param info additional information to be printed with the exception message
- * @return the given value.
- */
- public static String notEmpty(String value, String name, String info) {
- if (value == null) {
- throw new IllegalArgumentException(name + " cannot be null" + (info == null ? "" : ", " + info));
- }
- return notEmptyIfNotNull(value, name, info);
- }
-
- /**
- * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
- * throws an IllegalArgumentException.
- * @param list the list of strings.
- * @param name parameter name for the exception message.
- */
- public static Collection<String> notEmptyElements(Collection<String> list, String name) {
- notEmpty(list, name);
- for (String ele : list) {
- notEmpty(ele, String.format("list %s element %s", name, ele));
- }
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/classification/InterfaceAudience.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/classification/InterfaceAudience.java b/typesystem/src/main/java/org/apache/hadoop/metadata/classification/InterfaceAudience.java
deleted file mode 100755
index 41c5e93..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/classification/InterfaceAudience.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.classification;
-
-import java.lang.annotation.Documented;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-
-/**
- * Annotation to mark methods for consumption.
- */
-@InterfaceAudience.Public
-public class InterfaceAudience {
- private InterfaceAudience() {
- }
-
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface Private {
- }
-
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface LimitedPrivate {
- String[] value();
- }
-
- @Documented
- @Retention(RetentionPolicy.RUNTIME)
- public @interface Public {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IInstance.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IInstance.java
deleted file mode 100755
index 03b1ab3..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IInstance.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-import java.util.Map;
-
-/**
- * Represents a Struct or Trait or Object.
- */
-public interface IInstance {
-
- String getTypeName();
-
- Object get(String attrName) throws MetadataException;
-
- void set(String attrName, Object val) throws MetadataException;
-
- Map<String, Object> getValuesMap() throws MetadataException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IReferenceableInstance.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IReferenceableInstance.java
deleted file mode 100755
index 61be627..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IReferenceableInstance.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-
-/**
- * Represents and instance of a ClassType. These have identity.
- * Transient instances will have a UNASSIGNED identity.
- */
-public interface IReferenceableInstance extends IStruct {
-
- ImmutableList<String> getTraits();
-
- Id getId();
-
- IStruct getTrait(String typeName);
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IStruct.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IStruct.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IStruct.java
deleted file mode 100755
index d8116e0..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/IStruct.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem;
-
-/**
- * A marker interface for StructType and TraitType instances.
- */
-public interface IStruct extends IInstance {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedReferenceableInstance.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedReferenceableInstance.java
deleted file mode 100755
index e2680e6..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedReferenceableInstance.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem;
-
-public interface ITypedReferenceableInstance extends ITypedInstance, IReferenceableInstance {
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedStruct.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedStruct.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedStruct.java
deleted file mode 100755
index 9500ccf..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/ITypedStruct.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem;
-
-public interface ITypedStruct extends IStruct, ITypedInstance {
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Referenceable.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Referenceable.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Referenceable.java
deleted file mode 100755
index c3285c7..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Referenceable.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.classification.InterfaceAudience;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Represents a Class Instance that has not been associated with a FieldMapping.
- */
-public class Referenceable extends Struct implements IReferenceableInstance {
-
- private final Id id;
- private final ImmutableMap<String, IStruct> traits;
- private final ImmutableList<String> traitNames;
-
- public Referenceable(String typeName, String... traitNames) {
- super(typeName);
- id = new Id(typeName);
- this.traitNames = ImmutableList.copyOf(traitNames);
- ImmutableMap.Builder<String, IStruct> b = new ImmutableMap.Builder<String, IStruct>();
- for (String t : traitNames) {
- b.put(t, new Struct(t));
- }
- traits = b.build();
- }
-
- public Referenceable(String typeName, Map<String, Object> values) {
- super(typeName, values);
- id = new Id(typeName);
- traitNames = ImmutableList.of();
- traits = ImmutableMap.of();
- }
-
- public Referenceable(String guid, String typeName, Map<String, Object> values) {
- super(typeName, values);
- id = new Id(guid, 0, typeName);
- traitNames = ImmutableList.of();
- traits = ImmutableMap.of();
- }
-
- /**
- * Not public - only use during deserialization
- * @param guid
- * @param typeName
- * @param values
- */
- @InterfaceAudience.Private
- public Referenceable(String guid, String typeName, Map<String, Object> values,
- List<String> _traitNames,
- Map<String, IStruct> _traits) {
- super(typeName, values);
- id = new Id(guid, 0, typeName);
- traitNames = ImmutableList.copyOf(_traitNames);
- traits = ImmutableMap.copyOf(_traits);
- }
-
- @Override
- public ImmutableList<String> getTraits() {
- return traitNames;
- }
-
- @Override
- public Id getId() {
- return id;
- }
-
- @Override
- public IStruct getTrait(String typeName) {
- return traits.get(typeName);
- }
-}
[18/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftLexerTest.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftLexerTest.scala b/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftLexerTest.scala
deleted file mode 100755
index 2b3292f..0000000
--- a/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftLexerTest.scala
+++ /dev/null
@@ -1,525 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.thrift
-
-import org.junit.{Assert, Test}
-
-import scala.util.parsing.input.CharArrayReader
-
-class ThriftLexerTest {
-
- @Test def testSimple {
- val p = new ThriftParser
- val r = scan(p, """efg abc""")
- Assert.assertTrue(r.successful)
-
- }
-
- @Test def testStruct {
- val p = new ThriftParser
- val r = scan(p, """struct PartitionSpecWithSharedSD {
- 1: list<PartitionWithoutSD> partitions,
- 2: StorageDescriptor sd,
- }""")
- Assert.assertTrue(r.successful)
-
- }
-
- @Test def testTableStruct {
- val p = new ThriftParser
- val r = scan(p, """// table information
- struct Table {
- 1: string tableName, // name of the table
- 2: string dbName, // database name ('default')
- 3: string owner, // owner of this table
- 4: i32 createTime, // creation time of the table
- 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
- 6: i32 retention, // retention time
- 7: StorageDescriptor sd, // storage descriptor of the table
- 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
- 9: map<string, string> parameters, // to store comments or any other user level parameters
- 10: string viewOriginalText, // original view text, null for non-view
- 11: string viewExpandedText, // expanded view text, null for non-view
- 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
- 13: optional PrincipalPrivilegeSet privileges,
- 14: optional bool temporary=false
- }""")
- Assert.assertTrue(r.successful)
-
- }
-
- @Test def testIncorrectStruct {
- val p = new ThriftParser
- val r = scan(p, """// table information
- struct Table {
- | 1: string tableName, // name of the table
- | 2: string dbName
- }""")
- Assert.assertFalse(r.successful)
-
- }
-
- @Test def testNegativeInt {
- val p = new ThriftParser
- val r = scan(p, """-1""")
- Assert.assertTrue(r.successful)
-
- }
-
- @Test def testComment {
- val p = new ThriftParser
- val r = scan(p, """/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */""")
- }
-
- def scan(p: ThriftParser, str: String): p.lexical.ParseResult[_] = {
- val l = p.lexical
- var s: l.Input = new CharArrayReader(str.toCharArray)
- var r = (l.whitespace.? ~ l.token)(s)
- s = r.next
-
- while (r.successful && !s.atEnd) {
- s = r.next
- if (!s.atEnd) {
- r = (l.whitespace.? ~ l.token)(s)
- }
- }
- r.asInstanceOf[p.lexical.ParseResult[_]]
- }
-
- @Test def testService {
- val p = new ThriftParser
- val r = scan(p, """/**
- * This interface is live.
- */
- service ThriftHiveMetastore extends fb303.FacebookService
- {
- string getMetaConf(1:string key) throws(1:MetaException o1)
- void setMetaConf(1:string key, 2:string value) throws(1:MetaException o1)
-
- void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
- Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
- void drop_database(1:string name, 2:bool deleteData, 3:bool cascade) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
- list<string> get_databases(1:string pattern) throws(1:MetaException o1)
- list<string> get_all_databases() throws(1:MetaException o1)
- void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- // returns the type with given name (make seperate calls for the dependent types if needed)
- Type get_type(1:string name) throws(1:MetaException o1, 2:NoSuchObjectException o2)
- bool create_type(1:Type type) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
- bool drop_type(1:string type) throws(1:MetaException o1, 2:NoSuchObjectException o2)
- map<string, Type> get_type_all(1:string name)
- throws(1:MetaException o2)
-
- // Gets a list of FieldSchemas describing the columns of a particular table
- list<FieldSchema> get_fields(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3),
-
- // Gets a list of FieldSchemas describing both the columns and the partition keys of a particular table
- list<FieldSchema> get_schema(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
-
- // create a Hive table. Following fields must be set
- // tableName
- // database (only 'default' for now until Hive QL supports databases)
- // owner (not needed, but good to have for tracking purposes)
- // sd.cols (list of field schemas)
- // sd.inputFormat (SequenceFileInputFormat (binary like falcon tables or u_full) or TextInputFormat)
- // sd.outputFormat (SequenceFileInputFormat (binary) or TextInputFormat)
- // sd.serdeInfo.serializationLib (SerDe class name eg org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe
- // * See notes on DDL_TIME
- void create_table(1:Table tbl) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4)
- void create_table_with_environment_context(1:Table tbl,
- 2:EnvironmentContext environment_context)
- throws (1:AlreadyExistsException o1,
- 2:InvalidObjectException o2, 3:MetaException o3,
- 4:NoSuchObjectException o4)
- // drops the table and all the partitions associated with it if the table has partitions
- // delete data (including partitions) if deleteData is set to true
- void drop_table(1:string dbname, 2:string name, 3:bool deleteData)
- throws(1:NoSuchObjectException o1, 2:MetaException o3)
- void drop_table_with_environment_context(1:string dbname, 2:string name, 3:bool deleteData,
- 4:EnvironmentContext environment_context)
- throws(1:NoSuchObjectException o1, 2:MetaException o3)
- list<string> get_tables(1: string db_name, 2: string pattern) throws (1: MetaException o1)
- list<string> get_all_tables(1: string db_name) throws (1: MetaException o1)
-
- Table get_table(1:string dbname, 2:string tbl_name)
- throws (1:MetaException o1, 2:NoSuchObjectException o2)
- list<Table> get_table_objects_by_name(1:string dbname, 2:list<string> tbl_names)
- throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
-
- // Get a list of table names that match a filter.
- // The filter operators are LIKE, <, <=, >, >=, =, <>
- //
- // In the filter statement, values interpreted as strings must be enclosed in quotes,
- // while values interpreted as integers should not be. Strings and integers are the only
- // supported value types.
- //
- // The currently supported key names in the filter are:
- // Constants.HIVE_FILTER_FIELD_OWNER, which filters on the tables' owner's name
- // and supports all filter operators
- // Constants.HIVE_FILTER_FIELD_LAST_ACCESS, which filters on the last access times
- // and supports all filter operators except LIKE
- // Constants.HIVE_FILTER_FIELD_PARAMS, which filters on the tables' parameter keys and values
- // and only supports the filter operators = and <>.
- // Append the parameter key name to HIVE_FILTER_FIELD_PARAMS in the filter statement.
- // For example, to filter on parameter keys called "retention", the key name in the filter
- // statement should be Constants.HIVE_FILTER_FIELD_PARAMS + "retention"
- // Also, = and <> only work for keys that exist
- // in the tables. E.g., if you are looking for tables where key1 <> value, it will only
- // look at tables that have a value for the parameter key1.
- // Some example filter statements include:
- // filter = Constants.HIVE_FILTER_FIELD_OWNER + " like \".*test.*\" and " +
- // Constants.HIVE_FILTER_FIELD_LAST_ACCESS + " = 0";
- // filter = Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"30\" or " +
- // Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"90\""
- // @param dbName
- // The name of the database from which you will retrieve the table names
- // @param filterType
- // The type of filter
- // @param filter
- // The filter string
- // @param max_tables
- // The maximum number of tables returned
- // @return A list of table names that match the desired filter
- list<string> get_table_names_by_filter(1:string dbname, 2:string filter, 3:i16 max_tables=-1)
- throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
-
- // alter table applies to only future partitions not for existing partitions
- // * See notes on DDL_TIME
- void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
- void alter_table_with_environment_context(1:string dbname, 2:string tbl_name,
- 3:Table new_tbl, 4:EnvironmentContext environment_context)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
- // the following applies to only tables that have partitions
- // * See notes on DDL_TIME
- Partition add_partition(1:Partition new_part)
- throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- Partition add_partition_with_environment_context(1:Partition new_part,
- 2:EnvironmentContext environment_context)
- throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2,
- 3:MetaException o3)
- i32 add_partitions(1:list<Partition> new_parts)
- throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- i32 add_partitions_pspec(1:list<PartitionSpec> new_parts)
- throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
- throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- AddPartitionsResult add_partitions_req(1:AddPartitionsRequest request)
- throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- Partition append_partition_with_environment_context(1:string db_name, 2:string tbl_name,
- 3:list<string> part_vals, 4:EnvironmentContext environment_context)
- throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- Partition append_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name)
- throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- Partition append_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
- 3:string part_name, 4:EnvironmentContext environment_context)
- throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- bool drop_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:bool deleteData)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
- bool drop_partition_with_environment_context(1:string db_name, 2:string tbl_name,
- 3:list<string> part_vals, 4:bool deleteData, 5:EnvironmentContext environment_context)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
- bool drop_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name, 4:bool deleteData)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
- bool drop_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
- 3:string part_name, 4:bool deleteData, 5:EnvironmentContext environment_context)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
- DropPartitionsResult drop_partitions_req(1: DropPartitionsRequest req)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
-
- Partition get_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
- Partition exchange_partition(1:map<string, string> partitionSpecs, 2:string source_db,
- 3:string source_table_name, 4:string dest_db, 5:string dest_table_name)
- throws(1:MetaException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3,
- 4:InvalidInputException o4)
-
- Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals,
- 4: string user_name, 5: list<string> group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- Partition get_partition_by_name(1:string db_name 2:string tbl_name, 3:string part_name)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- // returns all the partitions for this table in reverse chronological order.
- // If max parts is given then it will return only that many.
- list<Partition> get_partitions(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
- list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1,
- 4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
-
- list<PartitionSpec> get_partitions_pspec(1:string db_name, 2:string tbl_name, 3:i32 max_parts=-1)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
-
- list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
- throws(1:MetaException o2)
-
- // get_partition*_ps methods allow filtering by a partial partition specification,
- // as needed for dynamic partitions. The values that are not restricted should
- // be empty strings. Nulls were considered (instead of "") but caused errors in
- // generated Python code. The size of part_vals may be smaller than the
- // number of partition columns - the unspecified values are considered the same
- // as "".
- list<Partition> get_partitions_ps(1:string db_name 2:string tbl_name
- 3:list<string> part_vals, 4:i16 max_parts=-1)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
- list<Partition> get_partitions_ps_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1,
- 5: string user_name, 6: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
-
- list<string> get_partition_names_ps(1:string db_name,
- 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- // get the partitions matching the given partition filter
- list<Partition> get_partitions_by_filter(1:string db_name 2:string tbl_name
- 3:string filter, 4:i16 max_parts=-1)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- // List partitions as PartitionSpec instances.
- list<PartitionSpec> get_part_specs_by_filter(1:string db_name 2:string tbl_name
- 3:string filter, 4:i32 max_parts=-1)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- // get the partitions matching the given partition filter
- // unlike get_partitions_by_filter, takes serialized hive expression, and with that can work
- // with any filter (get_partitions_by_filter only works if the filter can be pushed down to JDOQL.
- PartitionsByExprResult get_partitions_by_expr(1:PartitionsByExprRequest req)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- // get partitions give a list of partition names
- list<Partition> get_partitions_by_names(1:string db_name 2:string tbl_name 3:list<string> names)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- // changes the partition to the new partition object. partition is identified from the part values
- // in the new_part
- // * See notes on DDL_TIME
- void alter_partition(1:string db_name, 2:string tbl_name, 3:Partition new_part)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
-
- // change a list of partitions. All partitions are altered atomically and all
- // prehooks are fired together followed by all post hooks
- void alter_partitions(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
-
- void alter_partition_with_environment_context(1:string db_name,
- 2:string tbl_name, 3:Partition new_part,
- 4:EnvironmentContext environment_context)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
-
- // rename the old partition to the new partition object by changing old part values to the part values
- // in the new_part. old partition is identified from part_vals.
- // partition keys in new_part should be the same as those in old partition.
- void rename_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:Partition new_part)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
-
- // returns whether or not the partition name is valid based on the value of the config
- // hive.metastore.partition.name.whitelist.pattern
- bool partition_name_has_valid_characters(1:list<string> part_vals, 2:bool throw_exception)
- throws(1: MetaException o1)
-
- // gets the value of the configuration key in the metastore server. returns
- // defaultValue if the key does not exist. if the configuration key does not
- // begin with "hive", "mapred", or "hdfs", a ConfigValSecurityException is
- // thrown.
- string get_config_value(1:string name, 2:string defaultValue)
- throws(1:ConfigValSecurityException o1)
-
- // converts a partition name into a partition values array
- list<string> partition_name_to_vals(1: string part_name)
- throws(1: MetaException o1)
- // converts a partition name into a partition specification (a mapping from
- // the partition cols to the values)
- map<string, string> partition_name_to_spec(1: string part_name)
- throws(1: MetaException o1)
-
- void markPartitionForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
- 4:PartitionEventType eventType) throws (1: MetaException o1, 2: NoSuchObjectException o2,
- 3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
- 6: InvalidPartitionException o6)
- bool isPartitionMarkedForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
- 4: PartitionEventType eventType) throws (1: MetaException o1, 2:NoSuchObjectException o2,
- 3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
- 6: InvalidPartitionException o6)
-
- //index
- Index add_index(1:Index new_index, 2: Table index_table)
- throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
- void alter_index(1:string dbname, 2:string base_tbl_name, 3:string idx_name, 4:Index new_idx)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
- bool drop_index_by_name(1:string db_name, 2:string tbl_name, 3:string index_name, 4:bool deleteData)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
- Index get_index_by_name(1:string db_name 2:string tbl_name, 3:string index_name)
- throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- list<Index> get_indexes(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
- throws(1:NoSuchObjectException o1, 2:MetaException o2)
- list<string> get_index_names(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
- throws(1:MetaException o2)
-
- // column statistics interfaces
-
- // update APIs persist the column statistics object(s) that are passed in. If statistics already
- // exists for one or more columns, the existing statistics will be overwritten. The update APIs
- // validate that the dbName, tableName, partName, colName[] passed in as part of the ColumnStatistics
- // struct are valid, throws InvalidInputException/NoSuchObjectException if found to be invalid
- bool update_table_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
- 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
- bool update_partition_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
- 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
-
- // get APIs return the column statistics corresponding to db_name, tbl_name, [part_name], col_name if
- // such statistics exists. If the required statistics doesn't exist, get APIs throw NoSuchObjectException
- // For instance, if get_table_column_statistics is called on a partitioned table for which only
- // partition level column stats exist, get_table_column_statistics will throw NoSuchObjectException
- ColumnStatistics get_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
- (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidInputException o3, 4:InvalidObjectException o4)
- ColumnStatistics get_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name,
- 4:string col_name) throws (1:NoSuchObjectException o1, 2:MetaException o2,
- 3:InvalidInputException o3, 4:InvalidObjectException o4)
- TableStatsResult get_table_statistics_req(1:TableStatsRequest request) throws
- (1:NoSuchObjectException o1, 2:MetaException o2)
- PartitionsStatsResult get_partitions_statistics_req(1:PartitionsStatsRequest request) throws
- (1:NoSuchObjectException o1, 2:MetaException o2)
- AggrStats get_aggr_stats_for(1:PartitionsStatsRequest request) throws
- (1:NoSuchObjectException o1, 2:MetaException o2)
- bool set_aggr_stats_for(1:SetPartitionsStatsRequest request) throws
- (1:NoSuchObjectException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
-
-
- // delete APIs attempt to delete column statistics, if found, associated with a given db_name, tbl_name, [part_name]
- // and col_name. If the delete API doesn't find the statistics record in the metastore, throws NoSuchObjectException
- // Delete API validates the input and if the input is invalid throws InvalidInputException/InvalidObjectException.
- bool delete_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name, 4:string col_name) throws
- (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
- 4:InvalidInputException o4)
- bool delete_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
- (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
- 4:InvalidInputException o4)
-
- //
- // user-defined functions
- //
-
- void create_function(1:Function func)
- throws (1:AlreadyExistsException o1,
- 2:InvalidObjectException o2,
- 3:MetaException o3,
- 4:NoSuchObjectException o4)
-
- void drop_function(1:string dbName, 2:string funcName)
- throws (1:NoSuchObjectException o1, 2:MetaException o3)
-
- void alter_function(1:string dbName, 2:string funcName, 3:Function newFunc)
- throws (1:InvalidOperationException o1, 2:MetaException o2)
-
- list<string> get_functions(1:string dbName, 2:string pattern)
- throws (1:MetaException o1)
- Function get_function(1:string dbName, 2:string funcName)
- throws (1:MetaException o1, 2:NoSuchObjectException o2)
-
- //authorization privileges
-
- bool create_role(1:Role role) throws(1:MetaException o1)
- bool drop_role(1:string role_name) throws(1:MetaException o1)
- list<string> get_role_names() throws(1:MetaException o1)
- // Deprecated, use grant_revoke_role()
- bool grant_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type,
- 4:string grantor, 5:PrincipalType grantorType, 6:bool grant_option) throws(1:MetaException o1)
- // Deprecated, use grant_revoke_role()
- bool revoke_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type)
- throws(1:MetaException o1)
- list<Role> list_roles(1:string principal_name, 2:PrincipalType principal_type) throws(1:MetaException o1)
- GrantRevokeRoleResponse grant_revoke_role(1:GrantRevokeRoleRequest request) throws(1:MetaException o1)
-
- // get all role-grants for users/roles that have been granted the given role
- // Note that in the returned list of RolePrincipalGrants, the roleName is
- // redundant as it would match the role_name argument of this function
- GetPrincipalsInRoleResponse get_principals_in_role(1: GetPrincipalsInRoleRequest request) throws(1:MetaException o1)
-
- // get grant information of all roles granted to the given principal
- // Note that in the returned list of RolePrincipalGrants, the principal name,type is
- // redundant as it would match the principal name,type arguments of this function
- GetRoleGrantsForPrincipalResponse get_role_grants_for_principal(1: GetRoleGrantsForPrincipalRequest request) throws(1:MetaException o1)
-
- PrincipalPrivilegeSet get_privilege_set(1:HiveObjectRef hiveObject, 2:string user_name,
- 3: list<string> group_names) throws(1:MetaException o1)
- list<HiveObjectPrivilege> list_privileges(1:string principal_name, 2:PrincipalType principal_type,
- 3: HiveObjectRef hiveObject) throws(1:MetaException o1)
-
- // Deprecated, use grant_revoke_privileges()
- bool grant_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
- // Deprecated, use grant_revoke_privileges()
- bool revoke_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
- GrantRevokePrivilegeResponse grant_revoke_privileges(1:GrantRevokePrivilegeRequest request) throws(1:MetaException o1);
-
- // this is used by metastore client to send UGI information to metastore server immediately
- // after setting up a connection.
- list<string> set_ugi(1:string user_name, 2:list<string> group_names) throws (1:MetaException o1)
-
- //Authentication (delegation token) interfaces
-
- // get metastore server delegation token for use from the map/reduce tasks to authenticate
- // to metastore server
- string get_delegation_token(1:string token_owner, 2:string renewer_kerberos_principal_name)
- throws (1:MetaException o1)
-
- // method to renew delegation token obtained from metastore server
- i64 renew_delegation_token(1:string token_str_form) throws (1:MetaException o1)
-
- // method to cancel delegation token obtained from metastore server
- void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1)
-
- // Transaction and lock management calls
- // Get just list of open transactions
- GetOpenTxnsResponse get_open_txns()
- // Get list of open transactions with state (open, aborted)
- GetOpenTxnsInfoResponse get_open_txns_info()
- OpenTxnsResponse open_txns(1:OpenTxnRequest rqst)
- void abort_txn(1:AbortTxnRequest rqst) throws (1:NoSuchTxnException o1)
- void commit_txn(1:CommitTxnRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
- LockResponse lock(1:LockRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
- LockResponse check_lock(1:CheckLockRequest rqst)
- throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2, 3:NoSuchLockException o3)
- void unlock(1:UnlockRequest rqst) throws (1:NoSuchLockException o1, 2:TxnOpenException o2)
- ShowLocksResponse show_locks(1:ShowLocksRequest rqst)
- void heartbeat(1:HeartbeatRequest ids) throws (1:NoSuchLockException o1, 2:NoSuchTxnException o2, 3:TxnAbortedException o3)
- HeartbeatTxnRangeResponse heartbeat_txn_range(1:HeartbeatTxnRangeRequest txns)
- void compact(1:CompactionRequest rqst)
- ShowCompactResponse show_compact(1:ShowCompactRequest rqst)
- }""")
- Assert.assertTrue(r.successful)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParserTest.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParserTest.scala b/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParserTest.scala
deleted file mode 100755
index 486c875..0000000
--- a/tools/src/test/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParserTest.scala
+++ /dev/null
@@ -1,772 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.thrift
-
-import com.google.gson.JsonParser
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.{write => swrite}
-import org.json4s.{NoTypeHints, _}
-import org.junit.{Assert, Test}
-
-import scala.io.Source
-import scala.reflect.ClassTag
-
-/**
- * Copied from
- * [[https://github.com/json4s/json4s/blob/master/ext/src/main/scala/org/json4s/ext/EnumSerializer.scala json4s github]]
- * to avoid dependency on json4s-ext.
- */
-class EnumNameSerializer[E <: Enumeration: ClassTag](enum: E) extends Serializer[E#Value] {
- import JsonDSL._
-
- val EnumerationClass = classOf[E#Value]
-
- def deserialize(implicit format: Formats): PartialFunction[(TypeInfo, JValue), E#Value] = {
- case (t@TypeInfo(EnumerationClass, _), json) if (isValid(json)) => {
- json match {
- case JString(value) => enum.withName(value)
- case value => throw new MappingException("Can't convert " +
- value + " to " + EnumerationClass)
- }
- }
- }
-
- private[this] def isValid(json: JValue) = json match {
- case JString(value) if (enum.values.exists(_.toString == value)) => true
- case _ => false
- }
-
- def serialize(implicit format: Formats): PartialFunction[Any, JValue] = {
- case i: E#Value => i.toString
- }
-}
-
-class ThriftParserTest {
-
- @Test def testSimple {
- var p = new ThriftParser
- val parser = new JsonParser
-
- var td: Option[ThriftDef] = p( """include "share/fb303/if/fb303.thrift"
-
- namespace java org.apache.hadoop.hive.metastore.api
- namespace php metastore
- namespace cpp Apache.Hadoop.Hive
- """)
-
- val parsed = parser.parse(toJson(td.get))
- val sample = parser.parse( """{
- "includes":[{
- "value":"share/fb303/if/fb303.thrift"
- }],
- "cppIncludes":[],
- "namespaces":[{
- "lang":"",
- "name":"Apache.Hadoop.Hive",
- "otherLang":"cpp"
- },{
- "lang":"",
- "name":"metastore",
- "otherLang":"php"
- },{
- "lang":"",
- "name":"org.apache.hadoop.hive.metastore.api",
- "otherLang":"java"
- }],
- "constants":[],
- "typedefs":[],
- "enums":[],
- "senums":[],
- "structs":[],
- "unions":[],
- "xceptions":[],
- "services":[]
-}""")
-
- Assert.assertEquals(parsed.toString, sample.toString)
- }
-
- @Test def testStruct {
- val p = new ThriftParser
- val parser = new JsonParser
-
- var td: Option[ThriftDef] = p( """struct PartitionSpecWithSharedSD {
- 1: list<PartitionWithoutSD> partitions,
- 2: StorageDescriptor sd
- }""")
-
- val parsed = parser.parse(toJson(td.get))
-
- val sample = parser.parse( """{
- "includes":[],
- "cppIncludes":[],
- "namespaces":[],
- "constants":[],
- "typedefs":[],
- "enums":[],
- "senums":[],
- "structs":[{
- "name":"PartitionSpecWithSharedSD",
- "xsdAll":false,
- "fields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "elemType":{
- "name":"PartitionWithoutSD"
- }
- },
- "name":"partitions",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "name":"StorageDescriptor"
- },
- "name":"sd",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- }],
- "unions":[],
- "xceptions":[],
- "services":[]
-}""")
-
- Assert.assertEquals(parsed.toString, sample.toString)
- }
-
- def toJson(td: ThriftDef) = {
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new EnumNameSerializer(BASE_TYPES) +
- new EnumNameSerializer(THRIFT_LANG)
- val ser = swrite(td)
- pretty(render(parse(ser)))
- }
-
- @Test def testTableStruct {
- val p = new ThriftParser
- val parser = new JsonParser
-
- var td: Option[ThriftDef] = p( """// table information
- struct Table {
- 1: string tableName, // name of the table
- 2: string dbName, // database name ('default')
- 3: string owner, // owner of this table
- 4: i32 createTime, // creation time of the table
- 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
- 6: i32 retention, // retention time
- 7: StorageDescriptor sd, // storage descriptor of the table
- 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
- 9: map<string, string> parameters, // to store comments or any other user level parameters
- 10: string viewOriginalText, // original view text, null for non-view
- 11: string viewExpandedText, // expanded view text, null for non-view
- 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
- 13: optional PrincipalPrivilegeSet privileges,
- 14: optional bool temporary=false
- }""")
-
- val parsed = parser.parse(toJson(td.get))
- val sample = parser.parse( """{
- "includes":[],
- "cppIncludes":[],
- "namespaces":[],
- "constants":[],
- "typedefs":[],
- "enums":[],
- "senums":[],
- "structs":[{
- "name":"Table",
- "xsdAll":false,
- "fields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"tableName",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"dbName",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":3
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"owner",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":4
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"i32"
- },
- "name":"createTime",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":5
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"i32"
- },
- "name":"lastAccessTime",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":6
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"i32"
- },
- "name":"retention",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":7
- },
- "requiredNess":false,
- "fieldType":{
- "name":"StorageDescriptor"
- },
- "name":"sd",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":8
- },
- "requiredNess":false,
- "fieldType":{
- "elemType":{
- "name":"FieldSchema"
- }
- },
- "name":"partitionKeys",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":9
- },
- "requiredNess":false,
- "fieldType":{
- "keyType":{
- "typ":"string"
- },
- "valueType":{
- "typ":"string"
- }
- },
- "name":"parameters",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":10
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"viewOriginalText",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":11
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"viewExpandedText",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":12
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"tableType",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":13
- },
- "requiredNess":false,
- "fieldType":{
- "name":"PrincipalPrivilegeSet"
- },
- "name":"privileges",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":14
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"bool"
- },
- "name":"temporary",
- "fieldValue":{
- "value":"false"
- },
- "xsdOptional":false,
- "xsdNillable":false
- }]
- }],
- "unions":[],
- "xceptions":[],
- "services":[]
-}""")
-
- Assert.assertEquals(parsed.toString, sample.toString)
- }
-
- @Test def testHiveThrift {
- val p = new ThriftParser
- val is = getClass().getResourceAsStream("/test.thrift")
- val src: Source = Source.fromInputStream(is)
- val t: String = src.getLines().mkString("\n")
- var td: Option[ThriftDef] = p(t)
- Assert.assertTrue(td.isDefined)
- //println(toJson(td.get))
- }
-
- @Test def testService {
- val p = new ThriftParser
- val parser = new JsonParser
-
- var td: Option[ThriftDef] = p( """/**
- * This interface is live.
- */
- service ThriftHiveMetastore extends fb303.FacebookService
- {
- string getMetaConf(1:string key) throws(1:MetaException o1)
- void setMetaConf(1:string key, 2:string value) throws(1:MetaException o1)
-
- void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
- Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
- void drop_database(1:string name, 2:bool deleteData, 3:bool cascade) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
- list<string> get_databases(1:string pattern) throws(1:MetaException o1)
- list<string> get_all_databases() throws(1:MetaException o1)
- void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
-
- }""")
-
- val parsed = parser.parse(toJson(td.get))
- val sample = parser.parse( """{
- "includes":[],
- "cppIncludes":[],
- "namespaces":[],
- "constants":[],
- "typedefs":[],
- "enums":[],
- "senums":[],
- "structs":[],
- "unions":[],
- "xceptions":[],
- "services":[{
- "name":"ThriftHiveMetastore",
- "superName":"fb303.FacebookService",
- "functions":[{
- "oneway":false,
- "returnType":{
- "typ":"string"
- },
- "name":"getMetaConf",
- "parameters":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"key",
- "xsdOptional":false,
- "xsdNillable":false
- }],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- },{
- "oneway":false,
- "returnType":{
-
- },
- "name":"setMetaConf",
- "parameters":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"key",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"value",
- "xsdOptional":false,
- "xsdNillable":false
- }],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- },{
- "oneway":false,
- "returnType":{
-
- },
- "name":"create_database",
- "parameters":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"Database"
- },
- "name":"database",
- "xsdOptional":false,
- "xsdNillable":false
- }],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"AlreadyExistsException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "name":"InvalidObjectException"
- },
- "name":"o2",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":3
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o3",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- },{
- "oneway":false,
- "returnType":{
- "name":"Database"
- },
- "name":"get_database",
- "parameters":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"name",
- "xsdOptional":false,
- "xsdNillable":false
- }],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"NoSuchObjectException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o2",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- },{
- "oneway":false,
- "returnType":{
-
- },
- "name":"drop_database",
- "parameters":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"name",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"bool"
- },
- "name":"deleteData",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":3
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"bool"
- },
- "name":"cascade",
- "xsdOptional":false,
- "xsdNillable":false
- }],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"NoSuchObjectException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "name":"InvalidOperationException"
- },
- "name":"o2",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":3
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o3",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- },{
- "oneway":false,
- "returnType":{
- "elemType":{
- "typ":"string"
- }
- },
- "name":"get_databases",
- "parameters":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"pattern",
- "xsdOptional":false,
- "xsdNillable":false
- }],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- },{
- "oneway":false,
- "returnType":{
- "elemType":{
- "typ":"string"
- }
- },
- "name":"get_all_databases",
- "parameters":[],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- },{
- "oneway":false,
- "returnType":{
-
- },
- "name":"alter_database",
- "parameters":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "typ":"string"
- },
- "name":"dbname",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "name":"Database"
- },
- "name":"db",
- "xsdOptional":false,
- "xsdNillable":false
- }],
- "throwFields":[{
- "id":{
- "value":1
- },
- "requiredNess":false,
- "fieldType":{
- "name":"MetaException"
- },
- "name":"o1",
- "xsdOptional":false,
- "xsdNillable":false
- },{
- "id":{
- "value":2
- },
- "requiredNess":false,
- "fieldType":{
- "name":"NoSuchObjectException"
- },
- "name":"o2",
- "xsdOptional":false,
- "xsdNillable":false
- }]
- }]
- }]
-}""")
-
- Assert.assertEquals(parsed.toString, sample.toString)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/MetadataException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/MetadataException.java b/typesystem/src/main/java/org/apache/atlas/MetadataException.java
new file mode 100755
index 0000000..ac7a27f
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/MetadataException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+/**
+ * Base Exception class for metadata API.
+ */
+public class MetadataException extends Exception {
+
+ public MetadataException() {
+ }
+
+ public MetadataException(String message) {
+ super(message);
+ }
+
+ public MetadataException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public MetadataException(Throwable cause) {
+ super(cause);
+ }
+
+ public MetadataException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/ParamChecker.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/ParamChecker.java b/typesystem/src/main/java/org/apache/atlas/ParamChecker.java
new file mode 100644
index 0000000..e900ba3
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/ParamChecker.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+public class ParamChecker {
+
+ /**
+ * Check that a value is not null. If null throws an IllegalArgumentException.
+ *
+ * @param obj value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static <T> T notNull(T obj, String name) {
+ if (obj == null) {
+ throw new IllegalArgumentException(name + " cannot be null");
+ }
+ return obj;
+ }
+
+ /**
+ * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
+ * throws an IllegalArgumentException.
+ * @param list the list of T.
+ * @param name parameter name for the exception message.
+ */
+ public static <T> Collection<T> notNullElements(Collection<T> list, String name) {
+ notEmpty(list, name);
+ for (T ele : list) {
+ notNull(ele, String.format("Collection %s element %s", name, ele));
+ }
+ return list;
+ }
+
+ /**
+ * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
+ * throws an IllegalArgumentException.
+ * @param array the array of T.
+ * @param name parameter name for the exception message.
+ */
+ public static <T> T[] notNullElements(T[] array, String name) {
+ notEmpty(Arrays.asList(array), name);
+ for (T ele : array) {
+ notNull(ele, String.format("Collection %s element %s", name, ele));
+ }
+ return array;
+ }
+
+ /**
+ * Check that a list is not null and not empty.
+ * @param list the list of T.
+ * @param name parameter name for the exception message.
+ */
+ public static <T> Collection<T> notEmpty(Collection<T> list, String name) {
+ notNull(list, name);
+ if (list.isEmpty()) {
+ throw new IllegalArgumentException(String.format("Collection %s is empty", name));
+ }
+ return list;
+ }
+
+ /**
+ * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static String notEmpty(String value, String name) {
+ return notEmpty(value, name, null);
+ }
+
+ /**
+ * Check that a string is not empty if its not null.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static String notEmptyIfNotNull(String value, String name) {
+ return notEmptyIfNotNull(value, name, null);
+ }
+
+ /**
+ * Check that a string is not empty if its not null.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static String notEmptyIfNotNull(String value, String name, String info) {
+ if (value == null) {
+ return value;
+ }
+
+ if (value.trim().length() == 0) {
+ throw new IllegalArgumentException(name + " cannot be empty" + (info == null ? "" : ", " + info));
+ }
+ return value.trim();
+ }
+
+ /**
+ * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @param info additional information to be printed with the exception message
+ * @return the given value.
+ */
+ public static String notEmpty(String value, String name, String info) {
+ if (value == null) {
+ throw new IllegalArgumentException(name + " cannot be null" + (info == null ? "" : ", " + info));
+ }
+ return notEmptyIfNotNull(value, name, info);
+ }
+
+ /**
+ * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
+ * throws an IllegalArgumentException.
+ * @param list the list of strings.
+ * @param name parameter name for the exception message.
+ */
+ public static Collection<String> notEmptyElements(Collection<String> list, String name) {
+ notEmpty(list, name);
+ for (String ele : list) {
+ notEmpty(ele, String.format("list %s element %s", name, ele));
+ }
+ return list;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/classification/InterfaceAudience.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/classification/InterfaceAudience.java b/typesystem/src/main/java/org/apache/atlas/classification/InterfaceAudience.java
new file mode 100755
index 0000000..ae162ac
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/classification/InterfaceAudience.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.classification;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Annotation to mark methods for consumption.
+ */
+@InterfaceAudience.Public
+public class InterfaceAudience {
+ private InterfaceAudience() {
+ }
+
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Private {
+ }
+
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface LimitedPrivate {
+ String[] value();
+ }
+
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Public {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
new file mode 100755
index 0000000..dc08833
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem;
+
+import org.apache.atlas.MetadataException;
+
+import java.util.Map;
+
+/**
+ * Represents a Struct or Trait or Object.
+ */
+public interface IInstance {
+
+ String getTypeName();
+
+ Object get(String attrName) throws MetadataException;
+
+ void set(String attrName, Object val) throws MetadataException;
+
+ Map<String, Object> getValuesMap() throws MetadataException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/IReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/IReferenceableInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/IReferenceableInstance.java
new file mode 100755
index 0000000..9285014
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/IReferenceableInstance.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.typesystem.persistence.Id;
+
+/**
+ * Represents and instance of a ClassType. These have identity.
+ * Transient instances will have a UNASSIGNED identity.
+ */
+public interface IReferenceableInstance extends IStruct {
+
+ ImmutableList<String> getTraits();
+
+ Id getId();
+
+ IStruct getTrait(String typeName);
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java b/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java
new file mode 100755
index 0000000..e0f8576
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/IStruct.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem;
+
+/**
+ * A marker interface for StructType and TraitType instances.
+ */
+public interface IStruct extends IInstance {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedReferenceableInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedReferenceableInstance.java
new file mode 100755
index 0000000..6dae95c
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedReferenceableInstance.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem;
+
+public interface ITypedReferenceableInstance extends ITypedInstance, IReferenceableInstance {
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedStruct.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedStruct.java b/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedStruct.java
new file mode 100755
index 0000000..ced255c
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedStruct.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem;
+
+public interface ITypedStruct extends IStruct, ITypedInstance {
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java b/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
new file mode 100755
index 0000000..561d666
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.classification.InterfaceAudience;
+import org.apache.atlas.typesystem.persistence.Id;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents a Class Instance that has not been associated with a FieldMapping.
+ */
+public class Referenceable extends Struct implements IReferenceableInstance {
+
+ private final Id id;
+ private final ImmutableMap<String, IStruct> traits;
+ private final ImmutableList<String> traitNames;
+
+ public Referenceable(String typeName, String... traitNames) {
+ super(typeName);
+ id = new Id(typeName);
+ this.traitNames = ImmutableList.copyOf(traitNames);
+ ImmutableMap.Builder<String, IStruct> b = new ImmutableMap.Builder<String, IStruct>();
+ for (String t : traitNames) {
+ b.put(t, new Struct(t));
+ }
+ traits = b.build();
+ }
+
+ public Referenceable(String typeName, Map<String, Object> values) {
+ super(typeName, values);
+ id = new Id(typeName);
+ traitNames = ImmutableList.of();
+ traits = ImmutableMap.of();
+ }
+
+ public Referenceable(String guid, String typeName, Map<String, Object> values) {
+ super(typeName, values);
+ id = new Id(guid, 0, typeName);
+ traitNames = ImmutableList.of();
+ traits = ImmutableMap.of();
+ }
+
+ /**
+ * Not public - only use during deserialization
+ * @param guid
+ * @param typeName
+ * @param values
+ */
+ @InterfaceAudience.Private
+ public Referenceable(String guid, String typeName, Map<String, Object> values,
+ List<String> _traitNames,
+ Map<String, IStruct> _traits) {
+ super(typeName, values);
+ id = new Id(guid, 0, typeName);
+ traitNames = ImmutableList.copyOf(_traitNames);
+ traits = ImmutableMap.copyOf(_traits);
+ }
+
+ @Override
+ public ImmutableList<String> getTraits() {
+ return traitNames;
+ }
+
+ @Override
+ public Id getId() {
+ return id;
+ }
+
+ @Override
+ public IStruct getTrait(String typeName) {
+ return traits.get(typeName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java b/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
new file mode 100755
index 0000000..d57fbe0
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem;
+
+import org.apache.atlas.classification.InterfaceAudience;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Struct implements IStruct {
+
+ public final String typeName;
+ private final Map<String, Object> values;
+
+ public Struct(String typeName) {
+ this.typeName = typeName;
+ values = new HashMap<>();
+ }
+
+ @InterfaceAudience.Private
+ public Struct(String typeName, Map<String, Object> values) {
+ this(typeName);
+ if (values != null) {
+ this.values.putAll(values);
+ }
+ }
+
+ @Override
+ public String getTypeName() {
+ return typeName;
+ }
+
+ @Override
+ public Object get(String attrName) {
+ return values.get(attrName);
+ }
+
+ @Override
+ public void set(String attrName, Object value) {
+ values.put(attrName, value);
+ }
+
+ @Override
+ public Map<String, Object> getValuesMap() {
+ return values;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
new file mode 100755
index 0000000..e327400
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.persistence;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.types.DownCastFieldMapping;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DownCastStructInstance implements IStruct {
+
+ public final String typeName;
+ public final DownCastFieldMapping fieldMapping;
+ public final IStruct backingInstance;
+
+ public DownCastStructInstance(String typeName, DownCastFieldMapping fieldMapping,
+ IStruct backingInstance) {
+ this.typeName = typeName;
+ this.fieldMapping = fieldMapping;
+ this.backingInstance = backingInstance;
+ }
+
+ @Override
+ public String getTypeName() {
+ return typeName;
+ }
+
+ @Override
+ public Object get(String attrName) throws MetadataException {
+ return fieldMapping.get(this, attrName);
+ }
+
+ @Override
+ public void set(String attrName, Object val) throws MetadataException {
+ fieldMapping.set(this, attrName, val);
+ }
+
+ /*
+ * Use only for json serialization
+ * @nonpublic
+ */
+ @Override
+ public Map<String, Object> getValuesMap() throws MetadataException {
+
+ Map<String,Object> m = new HashMap<>();
+ for (String attr : fieldMapping.fieldNameMap.keySet()) {
+ m.put(attr, get(attr));
+ }
+ return m;
+ }
+}
+
+
[44/58] [abbrv] incubator-atlas git commit: Merge branch 'master'
into dal
Posted by ve...@apache.org.
Merge branch 'master' into dal
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/0ed1a1d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/0ed1a1d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/0ed1a1d0
Branch: refs/heads/master
Commit: 0ed1a1d0c927d5f00149f3970387dbbd90497c35
Parents: b286488 189ba06
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Tue Jun 9 17:15:25 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Tue Jun 9 17:15:25 2015 +0530
----------------------------------------------------------------------
.../atlas/repository/graph/GraphBackedMetadataRepository.java | 1 +
.../src/main/java/org/apache/atlas/typesystem/types/DataTypes.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[48/58] [abbrv] incubator-atlas git commit: Merge branch
'apache-local' into dal
Posted by ve...@apache.org.
Merge branch 'apache-local' into dal
Conflicts:
repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/2fbe8d9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/2fbe8d9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/2fbe8d9b
Branch: refs/heads/master
Commit: 2fbe8d9b233b05fd61e46186025851ea3a7a0841
Parents: f4579b6 2febe21
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Fri Jun 12 17:08:17 2015 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Fri Jun 12 17:08:17 2015 -0700
----------------------------------------------------------------------
InstallationSteps.txt | 21 +-
LICENSE.txt | 186 +-
NOTICE.txt | 2 +-
README.txt | 2 +-
addons/falcon-bridge/pom.xml | 82 -
.../org/apache/atlas/falcon/FalconImporter.java | 144 -
.../apache/atlas/falcon/FalconTypeSystem.java | 169 -
.../apache/atlas/falcon/FalconImporterTest.java | 108 -
.../atlas/falcon/FalconTypeSystemTest.java | 35 -
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 45 +-
.../org/apache/atlas/hive/hook/HiveHook.java | 52 +-
.../hive/model/HiveDataModelGenerator.java | 358 +-
.../src/site/twiki/Bridge-Hive.twiki | 14 +-
.../atlas/hive/hook/BaseSSLAndKerberosTest.java | 18 +-
.../org/apache/atlas/hive/hook/HiveHookIT.java | 33 +-
.../hook/NegativeSSLAndKerberosHiveHookIT.java | 3 +-
.../hive/hook/SSLAndKerberosHiveHookIT.java | 8 +-
.../apache/atlas/hive/hook/SSLHiveHookIT.java | 26 +-
.../metadata-bridge-core/.gitignore | 17 -
.../metadata-bridge-core/pom.xml | 75 -
.../apache/hadoop/metadata/bridge/ABridge.java | 166 -
.../hadoop/metadata/bridge/AEntityBean.java | 23 -
.../hadoop/metadata/bridge/BridgeException.java | 34 -
.../hadoop/metadata/bridge/BridgeManager.java | 153 -
.../metadata/bridge/BridgeTypeBootstrapper.java | 106 -
.../apache/hadoop/metadata/bridge/IBridge.java | 25 -
.../bridge/hivelineage/HiveLineageBridge.java | 35 -
.../bridge/hivelineage/hook/HiveLineage.java | 349 -
.../bridge/hivestructure/HiveMetaImporter.java | 221 -
.../hivestructure/HiveStructureBridge.java | 121 -
.../metadata/bridge/module/BridgeModule.java | 89 -
.../web/resources/HiveLineageResource.java | 82 -
.../main/resources/bridge-manager.properties | 22 -
.../metadata/bridge/BridgeManagerTest.java | 58 -
.../metadata/bridge/TestBridgeModule.java | 34 -
.../metadata/bridge/TestGenericBridges.java | 25 -
.../hivelineage/TestHiveLineageBridge.java | 86 -
.../test/resources/bridge-manager.properties | 22 -
.../src/test/resources/graph.properties | 25 -
.../src/test/resources/one.json | 48 -
.../resources/test-bridge-manager.properties | 22 -
.../src/test/resources/two.json | 48 -
.../metadata-bridge-hive/pom.xml | 112 -
.../bridge/hivelineage/hook/HiveLineage.java | 398 -
.../hivelineage/hook/HiveLineageInfo.java | 535 -
.../metadata/bridge/hivelineage/hook/Hook.java | 317 -
.../hivelineage/hook/HiveLineageInfoTest.java | 34 -
addons/metadata-bridge-parent/pom.xml | 35 -
.../main/java/org/apache/atlas/AtlasClient.java | 28 +-
.../atlas/security/SecureClientUtils.java | 35 +-
.../apache/atlas/security/BaseSecurityTest.java | 40 +-
dashboard/v1/Search.html | 126 -
dashboard/v1/css/bootstrap-responsive.css | 1109 -
dashboard/v1/css/bootstrap-responsive.min.css | 9 -
dashboard/v1/css/bootstrap-theme.css | 476 -
dashboard/v1/css/bootstrap-theme.css.map | 1 -
dashboard/v1/css/bootstrap-theme.min.css | 5 -
dashboard/v1/css/bootstrap.css | 6332 -----
dashboard/v1/css/bootstrap.css.map | 1 -
dashboard/v1/css/bootstrap.min.css | 5 -
dashboard/v1/css/heroic-features.css | 21 -
dashboard/v1/css/sticky-footer-navbar.css | 39 -
dashboard/v1/css/style.css | 93 -
.../v1/fonts/glyphicons-halflings-regular.eot | Bin 20335 -> 0 bytes
.../v1/fonts/glyphicons-halflings-regular.svg | 229 -
.../v1/fonts/glyphicons-halflings-regular.ttf | Bin 41280 -> 0 bytes
.../v1/fonts/glyphicons-halflings-regular.woff | Bin 23320 -> 0 bytes
.../v1/fonts/glyphicons-halflings-regular.woff2 | Bin 18028 -> 0 bytes
dashboard/v1/img/HWLOGO-g.png | Bin 7271 -> 0 bytes
dashboard/v1/img/HWLOGO.png | Bin 6762 -> 0 bytes
dashboard/v1/img/aetna-logo.png | Bin 5306 -> 0 bytes
dashboard/v1/img/elephant.png | Bin 184672 -> 0 bytes
dashboard/v1/img/glyphicons-halflings-white.png | Bin 8777 -> 0 bytes
dashboard/v1/img/glyphicons-halflings.png | Bin 12799 -> 0 bytes
dashboard/v1/img/graph1.1.jpg | Bin 38501 -> 0 bytes
dashboard/v1/img/graph1.jpg | Bin 15999 -> 0 bytes
dashboard/v1/img/hwlogo1.png | Bin 9263 -> 0 bytes
dashboard/v1/img/img-1.jpg | Bin 378640 -> 0 bytes
dashboard/v1/img/img-2.jpg | Bin 144750 -> 0 bytes
dashboard/v1/img/logo-1.png | Bin 4771 -> 0 bytes
dashboard/v1/img/logo-sm.png | Bin 2161 -> 0 bytes
dashboard/v1/img/mid-img.png | Bin 27460 -> 0 bytes
dashboard/v1/img/mid-img2.png | Bin 20836 -> 0 bytes
dashboard/v1/img/process.png | Bin 3204 -> 0 bytes
dashboard/v1/img/splash2.png | Bin 157503 -> 0 bytes
dashboard/v1/img/text.ico | Bin 4214 -> 0 bytes
dashboard/v1/index.html | 75 -
dashboard/v1/js/app.js | 38 -
dashboard/v1/js/attributes.html | 2 -
dashboard/v1/js/bootstrap.js | 2320 --
dashboard/v1/js/bootstrap.min.js | 7 -
dashboard/v1/js/controllers.js | 310 -
dashboard/v1/js/jquery.js | 4 -
dashboard/v1/js/npm.js | 13 -
dashboard/v1/lib/Angular/angular-route.min.js | 14 -
.../v1/lib/Angular/angular-route.min.js.map | 8 -
dashboard/v1/lib/Angular/angular.min.js | 212 -
dashboard/v1/lib/Angular/angular.min.js.map | 8 -
dashboard/v1/partials/attribute.html | 20 -
dashboard/v1/partials/lineage.html | 26 -
dashboard/v1/partials/search.html | 57 -
dashboard/v1/partials/wiki.html | 40 -
dashboard/v2/.bowerrc | 2 +-
dashboard/v2/.gitignore | 5 +-
dashboard/v2/.jshintignore | 18 +
dashboard/v2/.jshintrc | 18 +
dashboard/v2/README.md | 26 +-
dashboard/v2/bower.json | 3 +-
dashboard/v2/gruntfile.js | 109 +-
dashboard/v2/package.json | 11 +-
dashboard/v2/public/css/common.css | 46 +-
dashboard/v2/public/css/details.css | 21 +
dashboard/v2/public/css/lineage.css | 53 +-
.../v2/public/css/sticky-footer-navbar.css | 23 +
dashboard/v2/public/img/ApacheAtlasLogo.png | Bin 0 -> 3115 bytes
dashboard/v2/public/img/favicon.ico | Bin 2462 -> 3638 bytes
dashboard/v2/public/img/process.png | Bin
dashboard/v2/public/img/spinner.gif | Bin 0 -> 8275 bytes
dashboard/v2/public/img/tableicon.png | Bin 0 -> 478 bytes
dashboard/v2/public/index.html | 71 +
dashboard/v2/public/js/app.js | 9 +-
dashboard/v2/public/js/routes.js | 2 +-
.../public/modules/details/detailsController.js | 26 +-
.../v2/public/modules/details/detailsModule.js | 0
.../public/modules/details/detailsResource.js | 3 +-
.../v2/public/modules/details/detailsRoutes.js | 0
.../public/modules/details/views/attribute.html | 13 +-
.../public/modules/details/views/details.html | 38 +-
.../v2/public/modules/details/views/schema.html | 52 +
dashboard/v2/public/modules/home/homeModule.js | 2 +-
dashboard/v2/public/modules/home/homeRoutes.js | 31 -
dashboard/v2/public/modules/home/img/splash.png | Bin 157503 -> 275202 bytes
.../v2/public/modules/home/views/header.html | 4 +-
.../public/modules/lineage/lineageController.js | 312 +-
.../public/modules/lineage/lineageResource.js | 5 +-
.../public/modules/lineage/views/lineage.html | 8 +-
.../modules/navigation/navigationController.js | 31 +
.../modules/navigation/navigationModule.js | 21 +
.../modules/navigation/navigationResource.js | 36 +
.../modules/navigation/views/navigation.html | 24 +
.../public/modules/search/searchController.js | 95 +-
.../v2/public/modules/search/searchDirective.js | 34 +
.../v2/public/modules/search/searchResource.js | 10 +-
.../v2/public/modules/search/searchRoutes.js | 9 +-
.../v2/public/modules/search/views/search.html | 69 +-
.../modules/search/views/searchResult.html | 32 +-
.../modules/search/views/types/column.html | 21 +
.../public/modules/search/views/types/db.html | 22 +
.../public/modules/search/views/types/guid.html | 4 +-
.../modules/search/views/types/loadprocess.html | 22 +
.../modules/search/views/types/storagedesc.html | 21 +
.../modules/search/views/types/table.html | 22 +
.../public/modules/search/views/types/view.html | Bin 0 -> 1242 bytes
dashboard/v2/public/views/includes/foot.html | 37 -
dashboard/v2/public/views/includes/head.html | 46 -
dashboard/v2/public/views/index.html | 39 -
dashboard/v2/server.js | 18 +
dashboard/v2/server/config/config.js | 48 -
dashboard/v2/server/config/express.js | 146 -
dashboard/v2/server/routes/system/homeRoute.js | 29 -
dashboard/v3/css/bootstrap-responsive.css | 1109 -
dashboard/v3/css/bootstrap-responsive.min.css | 9 -
dashboard/v3/css/bootstrap-theme.css | 476 -
dashboard/v3/css/bootstrap-theme.css.map | 1 -
dashboard/v3/css/bootstrap-theme.min.css | 5 -
dashboard/v3/css/bootstrap.css | 5 -
dashboard/v3/css/bootstrap.css.map | 1 -
dashboard/v3/css/bootstrap.min.css | 5 -
dashboard/v3/css/heroic-features.css | 21 -
dashboard/v3/css/pagination.css | 536 -
dashboard/v3/css/sticky-footer-navbar.css | 39 -
dashboard/v3/css/style.css | 204 -
.../v3/fonts/glyphicons-halflings-regular.eot | Bin 20335 -> 0 bytes
.../v3/fonts/glyphicons-halflings-regular.svg | 229 -
.../v3/fonts/glyphicons-halflings-regular.ttf | Bin 41280 -> 0 bytes
.../v3/fonts/glyphicons-halflings-regular.woff | Bin 23320 -> 0 bytes
.../v3/fonts/glyphicons-halflings-regular.woff2 | Bin 18028 -> 0 bytes
dashboard/v3/img/arrow.png | Bin 305 -> 0 bytes
dashboard/v3/img/bg1.jpg | Bin 506 -> 0 bytes
dashboard/v3/img/bg2.jpg | Bin 506 -> 0 bytes
dashboard/v3/img/glyphicons-halflings-white.png | Bin 8777 -> 0 bytes
dashboard/v3/img/glyphicons-halflings.png | Bin 12799 -> 0 bytes
dashboard/v3/img/img1.jpg | Bin 853 -> 0 bytes
dashboard/v3/img/img1.png | Bin 1429 -> 0 bytes
dashboard/v3/img/img2.png | Bin 15218 -> 0 bytes
dashboard/v3/img/process.png | Bin 6925 -> 0 bytes
dashboard/v3/img/search_icon.jpg | Bin 893 -> 0 bytes
dashboard/v3/img/search_icon.png | Bin 2161 -> 0 bytes
dashboard/v3/img/tableicon.png | Bin 478 -> 0 bytes
dashboard/v3/img/topbg.jpg | Bin 13844 -> 0 bytes
dashboard/v3/index.html | 193 -
dashboard/v3/js/app.js | 63 -
dashboard/v3/js/config.json | 54 -
dashboard/v3/js/controllers.js | 1259 -
dashboard/v3/lib/angular-route.min.js | 14 -
dashboard/v3/lib/angular-route.min.js.map | 8 -
dashboard/v3/lib/angular-ui-router.js | 4232 ----
dashboard/v3/lib/angular-ui-router.min.js | 7 -
dashboard/v3/lib/angular.js | 21053 -----------------
dashboard/v3/lib/angular.min.js | 212 -
dashboard/v3/lib/angular.min.js.map | 8 -
dashboard/v3/lib/bootstrap.js | 7 -
dashboard/v3/lib/bootstrap.min.js | 7 -
dashboard/v3/lib/d3.tip.v0.6.3.js | 286 -
dashboard/v3/lib/ie-emulation-modes-warning.js | 51 -
.../v3/lib/ie10-viewport-bug-workaround.js | 22 -
dashboard/v3/lib/jquery.js | 4 -
dashboard/v3/lib/npm.js | 13 -
dashboard/v3/partials/search.html | 127 -
dashboard/v3/partials/wiki.html | 257 -
docs/license/d3-LICENSE.txt | 26 -
docs/license/jquery-LICENSE.txt | 19 -
docs/src/site/twiki/index.twiki | 4 +-
docs/src/site/twiki/security.twiki | 12 +-
pom.xml | 67 +-
repository/pom.xml | 5 +
.../java/org/apache/atlas/GraphTransaction.java | 6 +-
.../atlas/GraphTransactionInterceptor.java | 2 +-
.../apache/atlas/RepositoryMetadataModule.java | 14 +-
.../atlas/discovery/HiveLineageService.java | 67 +-
.../apache/atlas/discovery/SearchIndexer.java | 9 -
.../graph/DefaultGraphPersistenceStrategy.java | 124 +-
.../graph/GraphBackedDiscoveryService.java | 17 +-
.../atlas/listener/TypesChangeListener.java | 9 +-
.../org/apache/atlas/repository/Constants.java | 2 +-
.../repository/EntityNotFoundException.java | 2 +-
.../repository/IndexCreationException.java | 2 +-
.../apache/atlas/repository/IndexException.java | 3 +-
.../atlas/repository/MetadataRepository.java | 7 +-
.../atlas/repository/RepositoryException.java | 3 +-
.../graph/GraphBackedMetadataRepository.java | 679 +-
.../graph/GraphBackedSearchIndexer.java | 214 +-
.../atlas/repository/graph/GraphHelper.java | 44 +-
.../repository/memory/AttributeStores.java | 146 +-
.../atlas/repository/memory/ClassStore.java | 6 +-
.../memory/HierarchicalTypeStore.java | 11 +-
.../repository/memory/IAttributeStore.java | 3 +-
.../atlas/repository/memory/MemRepository.java | 26 +-
.../memory/ReplaceIdWithInstance.java | 20 +-
.../atlas/repository/memory/StructStore.java | 3 +-
.../atlas/repository/memory/TraitStore.java | 3 +-
.../typestore/GraphBackedTypeStore.java | 98 +-
.../atlas/services/DefaultMetadataService.java | 101 +-
.../apache/atlas/services/MetadataService.java | 7 +-
.../test/java/org/apache/atlas/TestUtils.java | 57 +-
.../GraphBackedDiscoveryServiceTest.java | 108 +-
.../atlas/discovery/HiveLineageServiceTest.java | 415 +-
.../org/apache/atlas/repository/BaseTest.java | 109 +-
.../GraphBackedMetadataRepositoryTest.java | 174 +-
.../graph/GraphRepoMapperScaleTest.java | 108 +-
.../atlas/repository/memory/EnumTest.java | 88 +-
.../repository/memory/InstanceE2ETest.java | 48 +-
.../atlas/repository/memory/StructTest.java | 3 +-
.../atlas/repository/memory/TraitTest.java | 20 +-
.../typestore/GraphBackedTypeStoreTest.java | 4 +-
src/build/findbugs-exclude.xml | 2 +-
src/main/assemblies/src-package.xml | 5 +
tools/pom.xml | 187 -
tools/src/main/resources/application.conf | 54 -
.../org/apache/atlas/tools/cli/Console.scala | 67 -
.../atlas/tools/dsl/DynamicTypedStruct.scala | 38 -
.../org/apache/atlas/tools/dsl/package.scala | 99 -
.../apache/atlas/tools/simpleserver/Main.scala | 59 -
.../tools/simpleserver/MetadataActor.scala | 121 -
.../tools/simpleserver/RestInterface.scala | 126 -
.../atlas/tools/thrift/ThriftParser.scala | 664 -
.../atlas/tools/thrift/ThriftTypesGen.scala | 316 -
tools/src/test/resources/test.thrift | 1146 -
.../org/apache/atlas/tools/dsl/DSLTest.scala | 241 -
.../tools/hive/HiveMockMetadataService.scala | 84 -
.../atlas/tools/thrift/ThriftLexerTest.scala | 525 -
.../atlas/tools/thrift/ThriftParserTest.scala | 772 -
.../org/apache/atlas/TypeNotFoundException.java | 2 +-
.../apache/atlas/typesystem/Referenceable.java | 5 +-
.../persistence/DownCastStructInstance.java | 5 +-
.../apache/atlas/typesystem/persistence/Id.java | 23 +-
.../atlas/typesystem/persistence/MapIds.java | 3 +-
.../persistence/ReferenceableInstance.java | 20 +-
.../typesystem/persistence/StructInstance.java | 187 +-
.../typesystem/types/AttributeDefinition.java | 47 +-
.../atlas/typesystem/types/AttributeInfo.java | 5 +-
.../atlas/typesystem/types/ClassType.java | 48 +-
.../atlas/typesystem/types/DataTypes.java | 40 +-
.../typesystem/types/DownCastFieldMapping.java | 3 +-
.../apache/atlas/typesystem/types/EnumType.java | 10 +-
.../typesystem/types/EnumTypeDefinition.java | 16 +-
.../atlas/typesystem/types/EnumValue.java | 16 +-
.../atlas/typesystem/types/FieldMapping.java | 15 +-
.../typesystem/types/HierarchicalType.java | 94 +-
.../types/HierarchicalTypeDefinition.java | 35 +-
.../atlas/typesystem/types/Multiplicity.java | 21 +-
.../typesystem/types/ObjectGraphTraversal.java | 11 +-
.../typesystem/types/ObjectGraphWalker.java | 17 +-
.../atlas/typesystem/types/StructType.java | 39 +-
.../typesystem/types/StructTypeDefinition.java | 16 +-
.../atlas/typesystem/types/TraitType.java | 6 +-
.../atlas/typesystem/types/TypeSystem.java | 213 +-
.../atlas/typesystem/types/TypeUtils.java | 14 +-
.../typesystem/types/TypedStructHandler.java | 16 +-
.../types/ValueConversionException.java | 11 +-
.../atlas/typesystem/types/utils/TypesUtil.java | 32 +-
.../typesystem/json/SerializationJavaTest.java | 50 +-
.../apache/atlas/typesystem/types/BaseTest.java | 120 +-
.../apache/atlas/typesystem/types/EnumTest.java | 82 +-
.../atlas/typesystem/types/StructTest.java | 3 +-
.../atlas/typesystem/types/TraitTest.java | 29 +-
.../typesystem/types/TypeInheritanceTest.java | 47 +-
.../atlas/typesystem/types/TypeSystemTest.java | 46 +-
.../atlas/typesystem/types/ValidationTest.java | 21 +-
webapp/pom.xml | 47 +-
webapp/src/main/java/org/apache/atlas/Main.java | 16 +-
.../org/apache/atlas/examples/QuickStart.java | 350 +-
.../atlas/util/CredentialProviderUtility.java | 33 +-
.../web/errors/LoggingExceptionMapper.java | 7 +-
.../apache/atlas/web/filters/AuditFilter.java | 17 +-
.../atlas/web/listeners/GuiceServletConfig.java | 8 +-
.../atlas/web/listeners/LoginProcessor.java | 13 +-
.../apache/atlas/web/params/AbstractParam.java | 8 +-
.../atlas/web/resources/AdminResource.java | 9 +-
.../atlas/web/resources/EntityResource.java | 88 +-
.../web/resources/HiveLineageResource.java | 36 +-
.../resources/MetadataDiscoveryResource.java | 56 +-
.../web/resources/RexsterGraphResource.java | 64 +-
.../atlas/web/resources/TypesResource.java | 25 +-
.../atlas/web/service/EmbeddedServer.java | 3 +-
.../atlas/web/service/SecureEmbeddedServer.java | 9 +-
.../apache/atlas/web/util/DateTimeHelper.java | 15 +-
.../org/apache/atlas/web/util/Servlets.java | 7 +-
.../atlas/CredentialProviderUtilityIT.java | 47 +-
.../MetadataAuthenticationKerberosFilterIT.java | 2 +-
.../MetadataAuthenticationSimpleFilterIT.java | 8 +-
.../atlas/web/listeners/LoginProcessorIT.java | 3 +-
.../web/resources/AdminJerseyResourceIT.java | 19 +-
.../atlas/web/resources/BaseResourceIT.java | 9 +-
.../web/resources/EntityJerseyResourceIT.java | 219 +-
.../resources/HiveLineageJerseyResourceIT.java | 211 +-
.../MetadataDiscoveryJerseyResourceIT.java | 60 +-
.../resources/RexsterGraphJerseyResourceIT.java | 15 +-
.../web/resources/TypesJerseyResourceIT.java | 89 +-
.../web/service/SecureEmbeddedServerIT.java | 4 +-
.../web/service/SecureEmbeddedServerITBase.java | 32 +-
341 files changed, 3971 insertions(+), 54917 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2fbe8d9b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2fbe8d9b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --cc repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 86a10f4,f494a4a..9556272
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@@ -348,12 -341,10 +341,11 @@@ public class DefaultMetadataService imp
onTraitAddedToEntity(guid, traitName);
}
- private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition) throws AtlasException {
+ private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition)
+ throws AtlasException {
try {
- Struct traitInstance = InstanceSerialization.fromJsonStruct(
- traitInstanceDefinition, true);
+ Struct traitInstance = InstanceSerialization.fromJsonStruct(traitInstanceDefinition, true);
final String entityTypeName = traitInstance.getTypeName();
ParamChecker.notEmpty(entityTypeName, "entity type cannot be null");
@@@ -443,5 -425,5 +427,4 @@@
public void unregisterListener(EntityChangeListener listener) {
entityChangeListeners.remove(listener);
}
--
-}
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2fbe8d9b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2fbe8d9b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
[45/58] [abbrv] incubator-atlas git commit: Merge branch 'master'
into dal
Posted by ve...@apache.org.
Merge branch 'master' into dal
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/fc190639
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/fc190639
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/fc190639
Branch: refs/heads/master
Commit: fc19063959b75c7dca565f84704c5eafda38b702
Parents: 0ed1a1d b10b34e
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Wed Jun 10 16:57:00 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Wed Jun 10 16:57:00 2015 +0530
----------------------------------------------------------------------
.../apache/hadoop/metadata/bridge/ABridge.java | 2 +-
.../atlas/GraphTransactionInterceptor.java | 2 +-
.../apache/atlas/RepositoryMetadataModule.java | 4 +-
.../apache/atlas/discovery/SearchIndexer.java | 8 ++
.../graph/DefaultGraphPersistenceStrategy.java | 1 +
.../repository/IndexCreationException.java | 41 ++++++++++
.../apache/atlas/repository/IndexException.java | 45 +++++++++++
.../graph/GraphBackedMetadataRepository.java | 25 ++++---
.../graph/GraphBackedSearchIndexer.java | 67 ++++++++++++-----
.../typestore/GraphBackedTypeStore.java | 1 +
.../atlas/services/DefaultMetadataService.java | 79 +++++++++++++-------
.../graph/GraphRepoMapperScaleTest.java | 74 ++++++++++++++++--
.../typestore/GraphBackedTypeStoreTest.java | 1 -
.../org/apache/atlas/TypeNotFoundException.java | 44 +++++++++++
.../atlas/typesystem/types/TypeSystem.java | 27 ++++---
.../apache/atlas/web/filters/AuditFilter.java | 5 +-
.../atlas/web/resources/EntityResource.java | 8 +-
.../web/resources/EntityJerseyResourceIT.java | 6 +-
.../web/resources/TypesJerseyResourceIT.java | 3 +
19 files changed, 354 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/fc190639/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --cc repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 07ba7f3,57e2d36..ce31412
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@@ -424,4 -446,5 +446,5 @@@ public class DefaultMetadataService imp
public void unregisterListener(EntityChangeListener listener) {
entityChangeListeners.remove(listener);
}
+
-}
+}
[54/58] [abbrv] incubator-atlas git commit: HDPDGI-81:Vishal:Display
querytext as tooltip for loadProcess
Posted by ve...@apache.org.
HDPDGI-81:Vishal:Display querytext as tooltip for loadProcess
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/eb9eb4cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/eb9eb4cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/eb9eb4cf
Branch: refs/heads/master
Commit: eb9eb4cf43b2a748cedbd168836f6354e6164272
Parents: 6c896a7
Author: Vishal Kadam <vi...@gmail.com>
Authored: Tue Jun 16 00:41:31 2015 -0400
Committer: Vishal Kadam <vi...@gmail.com>
Committed: Tue Jun 16 00:41:31 2015 -0400
----------------------------------------------------------------------
dashboard/v2/public/index.html | 1 +
.../public/modules/lineage/lineageController.js | 32 +++++++++++++++++---
2 files changed, 28 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb9eb4cf/dashboard/v2/public/index.html
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/index.html b/dashboard/v2/public/index.html
index 90356f3..b5ab634 100644
--- a/dashboard/v2/public/index.html
+++ b/dashboard/v2/public/index.html
@@ -65,6 +65,7 @@
<script src="lib/angular-ui-utils/ui-utils.js"></script>
<script src="lib/lodash/lodash.js"></script>
<script src="/lib/d3/d3.js"></script>
+<script src="/lib/d3-tip/index.js"></script>
<script src="js/app.min.js"></script>
</body>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/eb9eb4cf/dashboard/v2/public/modules/lineage/lineageController.js
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/modules/lineage/lineageController.js b/dashboard/v2/public/modules/lineage/lineageController.js
index 49ce91e..6539375 100755
--- a/dashboard/v2/public/modules/lineage/lineageController.js
+++ b/dashboard/v2/public/modules/lineage/lineageController.js
@@ -91,7 +91,7 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
nodes = {};
function getNode(guid) {
- var name, type;
+ var name, type, tip;
if (vertices.hasOwnProperty(guid)) {
name = vertices[guid].values.name;
type = vertices[guid].values.vertexId.values.typeName;
@@ -100,6 +100,7 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
if (typeof loadProcess !== "undefined") {
name = loadProcess.name;
type = loadProcess.typeName;
+ tip = loadProcess.tip;
} else {
name = 'Load Process';
type = 'Load Process';
@@ -108,7 +109,8 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
var vertex = {
guid: guid,
name: name,
- type: type
+ type: type,
+ tip: tip
};
if (!nodes.hasOwnProperty(guid)) {
nodes[guid] = vertex;
@@ -122,6 +124,7 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
if (value.id.id === guid) {
procesRes.name = value.values.name;
procesRes.typeName = value.typeName;
+ procesRes.tip = value.values.queryText;
}
});
return procesRes;
@@ -176,13 +179,21 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
return [d.y, d.x];
});
+ /* Initialize tooltip */
+ var tooltip = d3.tip()
+ .attr('class', 'd3-tip')
+ .html(function(d) {
+ return '<pre class="alert alert-success">' + d.tip + '</pre>';
+ });
+
var svg = element.select('svg')
.attr('width', width + margin.right + margin.left)
.attr('height', height + margin.top + margin.bottom)
+ /* Invoke the tip in the context of your visualization */
+ .call(tooltip)
.select('g')
-
- .attr('transform',
- 'translate(' + margin.left + ',' + margin.right + ')');
+ .attr('transform',
+ 'translate(' + margin.left + ',' + margin.right + ')');
//arrow
svg.append("svg:defs").append("svg:marker").attr("id", "arrow").attr("viewBox", "0 0 10 10").attr("refX", 26).attr("refY", 5).attr("markerUnits", "strokeWidth").attr("markerWidth", 6).attr("markerHeight", 9).attr("orient", "auto").append("svg:path").attr("d", "M 0 0 L 10 5 L 0 10 z");
@@ -217,10 +228,21 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
//return d.icon;
return d.type === 'Table' ? '../img/tableicon.png' : '../img/process.png';
})
+ .on('mouseover', function(d) {
+ if (d.type === 'LoadProcess') {
+ tooltip.show(d);
+ }
+ })
+ .on('mouseout', function(d) {
+ if (d.type === 'LoadProcess') {
+ tooltip.hide(d);
+ }
+ })
.attr("x", "-18px")
.attr("y", "-18px")
.attr("width", "34px")
.attr("height", "34px");
+
nodeEnter.append('text')
.attr('x', function(d) {
return d.children || d._children ?
[04/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java
deleted file mode 100755
index 541c2f4..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/TypesResource.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.sun.jersey.api.client.ClientResponse;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.services.MetadataService;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This class provides RESTful API for Types.
- *
- * A type is the description of any representable item;
- * e.g. a Hive table
- *
- * You could represent any meta model representing any domain using these types.
- */
-@Path("types")
-@Singleton
-public class TypesResource {
-
- private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
-
- private final MetadataService metadataService;
-
- static final String TYPE_ALL = "all";
-
- @Inject
- public TypesResource(MetadataService metadataService) {
- this.metadataService = metadataService;
- }
-
- /**
- * Submits a type definition corresponding to a given type representing a meta model of a
- * domain. Could represent things like Hive Database, Hive Table, etc.
- */
- @POST
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response submit(@Context HttpServletRequest request) {
- try {
- final String typeDefinition = Servlets.getRequestPayload(request);
- LOG.debug("Creating type with definition {} ", typeDefinition);
-
- JSONObject typesJson = metadataService.createType(typeDefinition);
- final JSONArray typesJsonArray = typesJson.getJSONArray(MetadataServiceClient.TYPES);
-
- JSONArray typesResponse = new JSONArray();
- for (int i = 0; i < typesJsonArray.length(); i++) {
- final String name = typesJsonArray.getString(i);
- typesResponse.put(new JSONObject() {{
- put(MetadataServiceClient.NAME, name);
- }});
- }
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.TYPES, typesResponse);
- return Response.status(ClientResponse.Status.CREATED).entity(response).build();
- } catch (MetadataException | IllegalArgumentException e) {
- LOG.error("Unable to persist types", e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to persist types", e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Fetch the complete definition of a given type name which is unique.
- *
- * @param typeName name of a type which is unique.
- */
- @GET
- @Path("{typeName}")
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response getDefinition(@Context HttpServletRequest request,
- @PathParam("typeName") String typeName) {
- try {
- final String typeDefinition = metadataService.getTypeDefinition(typeName);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.TYPENAME, typeName);
- response.put(MetadataServiceClient.DEFINITION, typeDefinition);
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
-
- return Response.ok(response).build();
- } catch (MetadataException e) {
- LOG.error("Unable to get type definition for type {}", typeName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (JSONException | IllegalArgumentException e) {
- LOG.error("Unable to get type definition for type {}", typeName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get type definition for type {}", typeName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Gets the list of trait type names registered in the type system.
- *
- * @param type type should be the name of enum
- * org.apache.hadoop.metadata.typesystem.types.DataTypes.TypeCategory
- * Typically, would be one of all, TRAIT, CLASS, ENUM, STRUCT
- * @return entity names response payload as json
- */
- @GET
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response getTypesByFilter(@Context HttpServletRequest request,
- @DefaultValue(TYPE_ALL) @QueryParam("type") String type) {
- try {
- List<String> result;
- if (TYPE_ALL.equals(type)) {
- result = metadataService.getTypeNamesList();
- } else {
- DataTypes.TypeCategory typeCategory = DataTypes.TypeCategory.valueOf(type);
- result = metadataService.getTypeNamesByCategory(typeCategory);
- }
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.RESULTS, new JSONArray(result));
- response.put(MetadataServiceClient.COUNT, result.size());
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
-
- return Response.ok(response).build();
- } catch (IllegalArgumentException | MetadataException ie) {
- LOG.error("Unsupported typeName while retrieving type list {}", type);
- throw new WebApplicationException(
- Servlets.getErrorResponse("Unsupported type " + type, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get types list", e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/service/EmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/service/EmbeddedServer.java
deleted file mode 100755
index 8c3b8ab..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/service/EmbeddedServer.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.service;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.bio.SocketConnector;
-import org.mortbay.jetty.webapp.WebAppContext;
-
-import java.io.IOException;
-
-/**
- * This class embeds a Jetty server and a connector.
- */
-public class EmbeddedServer {
- private static final int DEFAULT_BUFFER_SIZE = 16192;
-
- protected final Server server = new Server();
-
- public EmbeddedServer(int port, String path) throws IOException {
- Connector connector = getConnector(port);
- server.addConnector(connector);
-
- WebAppContext application = new WebAppContext(path, "/");
- server.setHandler(application);
- }
-
- public static EmbeddedServer newServer(int port, String path, boolean secure) throws IOException {
- if (secure) {
- return new SecureEmbeddedServer(port, path);
- } else {
- return new EmbeddedServer(port, path);
- }
- }
-
- protected Connector getConnector(int port) throws IOException {
- Connector connector = new SocketConnector();
- connector.setPort(port);
- connector.setHost("0.0.0.0");
-
- // this is to enable large header sizes when Kerberos is enabled with AD
- final Integer bufferSize = getBufferSize();
- connector.setHeaderBufferSize(bufferSize);
- connector.setRequestBufferSize(bufferSize);
-
- return connector;
- }
-
- private Integer getBufferSize() {
- try {
- PropertiesConfiguration configuration = new PropertiesConfiguration(
- "application.properties");
- return configuration.getInt("metadata.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
- } catch (ConfigurationException e) {
- // do nothing
- }
-
- return DEFAULT_BUFFER_SIZE;
- }
-
- public void start() throws Exception {
- server.start();
- server.join();
- }
-
- public void stop() throws Exception {
- server.stop();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServer.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServer.java
deleted file mode 100755
index 6e71366..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/service/SecureEmbeddedServer.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.service;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.security.SslSocketConnector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-import java.io.IOException;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-/**
- * This is a jetty server which requires client auth via certificates.
- */
-public class SecureEmbeddedServer extends EmbeddedServer {
-
- private static final Logger LOG = LoggerFactory.getLogger(SecureEmbeddedServer.class);
-
- public SecureEmbeddedServer(int port, String path) throws IOException {
- super(port, path);
- }
-
- protected Connector getConnector(int port) throws IOException {
- PropertiesConfiguration config = getConfiguration();
-
- SslSocketConnector connector = new SslSocketConnector();
- connector.setPort(port);
- connector.setHost("0.0.0.0");
- connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
- System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
- connector.setKeyPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
- connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
- System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
- connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
- connector.setPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
- connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
- return connector;
- }
-
- /**
- * Retrieves a password from a configured credential provider or prompts for the password and stores it in the
- * configured credential provider.
- * @param config application configuration
- * @param key the key/alias for the password.
- * @return the password.
- * @throws IOException
- */
- private String getPassword(PropertiesConfiguration config, String key) throws IOException {
-
- String password = null;
-
- String provider = config.getString(CERT_STORES_CREDENTIAL_PROVIDER_PATH);
- if (provider != null) {
- LOG.info("Attempting to retrieve password from configured credential provider path");
- Configuration c = new Configuration();
- c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
- CredentialProvider credentialProvider =
- CredentialProviderFactory.getProviders(c).get(0);
- CredentialProvider.CredentialEntry entry = credentialProvider.getCredentialEntry(key);
- if (entry == null) {
- throw new IOException(String.format("No credential entry found for %s. " +
- "Please create an entry in the configured credential provider", key));
- } else {
- password = String.valueOf(entry.getCredential());
- }
-
- } else {
- throw new IOException("No credential provider path configured for storage of certificate store passwords");
- }
-
- return password;
- }
-
- /**
- * Returns the application configuration.
- * @return
- */
- protected PropertiesConfiguration getConfiguration() {
- try {
- return new PropertiesConfiguration("application.properties");
- } catch (ConfigurationException e) {
- throw new RuntimeException("Unable to load configuration: application.properties");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/util/DateTimeHelper.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/util/DateTimeHelper.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/util/DateTimeHelper.java
deleted file mode 100755
index 6ad2f24..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/util/DateTimeHelper.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.util;
-
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.TimeZone;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Support function to parse and format date.
- */
-public final class DateTimeHelper {
-
- public static final String ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm'Z'";
- private static final String DATE_PATTERN =
- "(2\\d\\d\\d|19\\d\\d)-(0[1-9]|1[012])-(0[1-9]|1[0-9]|2[0-9]|3[01])T" +
- "([0-1][0-9]|2[0-3]):([0-5][0-9])Z";
- private static final Pattern PATTERN = Pattern.compile(DATE_PATTERN);
-
- private DateTimeHelper() {
- }
-
- public static String getTimeZoneId(TimeZone tz) {
- return tz.getID();
- }
-
- public static DateFormat getDateFormat() {
- DateFormat dateFormat = new SimpleDateFormat(ISO8601_FORMAT);
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- return dateFormat;
- }
-
- public static String formatDateUTC(Date date) {
- return (date != null) ? getDateFormat().format(date) : null;
- }
-
- public static Date parseDateUTC(String dateStr) {
- if (!validate(dateStr)) {
- throw new IllegalArgumentException(dateStr + " is not a valid UTC string");
- }
- try {
- return getDateFormat().parse(dateStr);
- } catch (ParseException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static String formatDateUTCToISO8601(final String dateString,
- final String dateStringFormat) {
-
- try {
- DateFormat dateFormat = new SimpleDateFormat(
- dateStringFormat.substring(0, dateString.length()));
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- return DateTimeHelper.formatDateUTC(dateFormat.parse(dateString));
- } catch (ParseException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Validate date format with regular expression.
- *
- * @param date date address for validation
- * @return true valid date fromat, false invalid date format
- */
- public static boolean validate(final String date) {
-
- Matcher matcher = PATTERN.matcher(date);
-
- if (matcher.matches()) {
-
- matcher.reset();
-
- if (matcher.find()) {
-
- int year = Integer.parseInt(matcher.group(1));
- String month = matcher.group(2);
- String day = matcher.group(3);
-
- if (day.equals("31")
- && (month.equals("4") || month.equals("6")
- || month.equals("9") || month.equals("11")
- || month.equals("04") || month.equals("06") || month.equals("09"))) {
- return false; // only 1,3,5,7,8,10,12 has 31 days
- } else if (month.equals("2") || month.equals("02")) {
- // leap year
- if (year % 4 == 0) {
- return !(day.equals("30") || day.equals("31"));
- } else {
- return !(day.equals("29") || day.equals("30") || day.equals("31"));
- }
- } else {
- return true;
- }
- } else {
- return false;
- }
- } else {
- return false;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/util/Servlets.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/util/Servlets.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/util/Servlets.java
deleted file mode 100755
index 47435a4..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/util/Servlets.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.util;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringEscapeUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.ParamChecker;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-
-/**
- * Utility functions for dealing with servlets.
- */
-public final class Servlets {
-
- private static final Logger LOG = LoggerFactory.getLogger(Servlets.class);
- private Servlets() {
- /* singleton */
- }
-
- public static final String JSON_MEDIA_TYPE = MediaType.APPLICATION_JSON + "; charset=UTF-8";
-
- /**
- * Returns the user of the given request.
- *
- * @param httpRequest an HTTP servlet request
- * @return the user
- */
- public static String getUserFromRequest(HttpServletRequest httpRequest) {
- String user = httpRequest.getRemoteUser();
- if (!StringUtils.isEmpty(user)) {
- return user;
- }
-
- user = httpRequest.getParameter("user.name"); // available in query-param
- if (!StringUtils.isEmpty(user)) {
- return user;
- }
-
- user = httpRequest.getHeader("Remote-User"); // backwards-compatibility
- if (!StringUtils.isEmpty(user)) {
- return user;
- }
-
- return null;
- }
-
- /**
- * Returns the URI of the given request.
- *
- * @param httpRequest an HTTP servlet request
- * @return the URI, including the query string
- */
- public static String getRequestURI(HttpServletRequest httpRequest) {
- final StringBuilder url = new StringBuilder(100).append(httpRequest.getRequestURI());
- if (httpRequest.getQueryString() != null) {
- url.append('?').append(httpRequest.getQueryString());
- }
-
- return url.toString();
- }
-
- /**
- * Returns the full URL of the given request.
- *
- * @param httpRequest an HTTP servlet request
- * @return the full URL, including the query string
- */
- public static String getRequestURL(HttpServletRequest httpRequest) {
- final StringBuilder url = new StringBuilder(100).append(httpRequest.getRequestURL());
- if (httpRequest.getQueryString() != null) {
- url.append('?').append(httpRequest.getQueryString());
- }
-
- return url.toString();
- }
-
- public static Response getErrorResponse(Throwable e, Response.Status status) {
- Response response = getErrorResponse(e.getMessage(), status);
- JSONObject responseJson = (JSONObject) response.getEntity();
- try {
- responseJson.put(MetadataServiceClient.STACKTRACE, printStackTrace(e));
- } catch (JSONException e1) {
- LOG.warn("Could not construct error Json rensponse", e1);
- }
- return response;
- }
-
- private static String printStackTrace(Throwable t) {
- StringWriter sw = new StringWriter();
- t.printStackTrace(new PrintWriter(sw));
- return sw.toString();
- }
-
- public static Response getErrorResponse(String message, Response.Status status) {
- JSONObject errorJson = new JSONObject();
- Object errorEntity = Servlets.escapeJsonString(message);
- try {
- errorJson.put(MetadataServiceClient.ERROR, errorEntity);
- errorEntity = errorJson;
- } catch (JSONException jsonE) {
- LOG.warn("Could not construct error Json rensponse", jsonE);
- }
- return Response
- .status(status)
- .entity(errorEntity)
- .type(JSON_MEDIA_TYPE)
- .build();
- }
-
- public static String getRequestPayload(HttpServletRequest request) throws IOException {
- StringWriter writer = new StringWriter();
- IOUtils.copy(request.getInputStream(), writer);
- return writer.toString();
- }
-
- public static String getRequestId() {
- return Thread.currentThread().getName();
- }
-
- public static String escapeJsonString(String inputStr) {
- ParamChecker.notNull(inputStr, "Input String cannot be null");
- return StringEscapeUtils.escapeJson(inputStr);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/CredentialProviderUtilityIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/CredentialProviderUtilityIT.java b/webapp/src/test/java/org/apache/atlas/CredentialProviderUtilityIT.java
new file mode 100755
index 0000000..2bb70d0
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/CredentialProviderUtilityIT.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas;
+
+import org.apache.atlas.security.SecurityProperties;
+import org.apache.atlas.util.CredentialProviderUtility;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ *
+ */
+public class CredentialProviderUtilityIT {
+
+ private char[] defaultPass = new char[]{'k', 'e', 'y', 'p', 'a', 's', 's'};
+
+ @Test
+ public void testEnterValidValues() throws Exception {
+ Path testPath = null;
+ try {
+ testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ new File(testPath.toUri().getPath()).delete();
+ final Path finalTestPath = testPath;
+ CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+ @Override
+ public void printf(String fmt, Object... params) {
+ System.out.print(String.format(fmt, params));
+ }
+
+ public String readLine(String fmt, Object ... args) {
+ return finalTestPath.toString();
+ }
+
+ @Override
+ public char[] readPassword(String fmt, Object ... args) {
+ return defaultPass;
+ }
+ };
+
+ CredentialProviderUtility.main(new String[] {});
+
+ String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+ Configuration conf = new Configuration(false);
+
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ CredentialProvider.CredentialEntry entry =
+ provider.getCredentialEntry(SecurityProperties.KEYSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ entry = provider.getCredentialEntry(SecurityProperties.TRUSTSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ entry = provider.getCredentialEntry(SecurityProperties.SERVER_CERT_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ }
+
+ protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry) {
+ assertCredentialEntryCorrect(entry, defaultPass);
+ }
+
+ protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry, char[] password) {
+ Assert.assertNotNull(entry);
+ Assert.assertEquals(entry.getCredential(), password);
+ }
+
+ @Test
+ public void testEnterEmptyValues() throws Exception {
+ Path testPath = null;
+ try {
+ testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ new File(testPath.toUri().getPath()).delete();
+ final Path finalTestPath = testPath;
+ CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+
+ private Random random = new Random();
+
+ @Override
+ public void printf(String fmt, Object... params) {
+ System.out.print(String.format(fmt, params));
+ }
+
+ public String readLine(String fmt, Object ... args) {
+ return finalTestPath.toString();
+ }
+
+ @Override
+ public char[] readPassword(String fmt, Object ... args) {
+ List<char[]> responses = new ArrayList<>();
+ responses.add(new char[0]);
+ responses.add(defaultPass);
+
+ int size = responses.size();
+ int item = random.nextInt(size);
+ return responses.get(item);
+ }
+ };
+
+ CredentialProviderUtility.main(new String[] {});
+
+ String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+ Configuration conf = new Configuration(false);
+
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ CredentialProvider.CredentialEntry entry =
+ provider.getCredentialEntry(SecurityProperties.KEYSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ entry = provider.getCredentialEntry(SecurityProperties.TRUSTSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ entry = provider.getCredentialEntry(SecurityProperties.SERVER_CERT_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ }
+
+ @Test
+ public void testEnterMismatchedValues() throws Exception {
+ Path testPath = null;
+ try {
+ testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ new File(testPath.toUri().getPath()).delete();
+ final Path finalTestPath = testPath;
+ CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+
+ int i = 0;
+ @Override
+ public void printf(String fmt, Object... params) {
+ System.out.print(String.format(fmt, params));
+ }
+
+ public String readLine(String fmt, Object ... args) {
+ return finalTestPath.toString();
+ }
+
+ @Override
+ public char[] readPassword(String fmt, Object ... args) {
+ List<char[]> responses = new ArrayList<>();
+ responses.add(defaultPass);
+ responses.add(new char[] {'b', 'a', 'd', 'p', 'a', 's', 's'});
+ responses.add(defaultPass);
+
+ int item = i % 3;
+ i++;
+ return responses.get(item);
+ }
+ };
+
+ CredentialProviderUtility.main(new String[] {});
+
+ String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+ Configuration conf = new Configuration(false);
+
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ CredentialProvider.CredentialEntry entry =
+ provider.getCredentialEntry(SecurityProperties.KEYSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ entry = provider.getCredentialEntry(SecurityProperties.TRUSTSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ entry = provider.getCredentialEntry(SecurityProperties.SERVER_CERT_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry);
+ }
+
+ @Test
+ public void testOverwriteValues() throws Exception {
+ Path testPath = null;
+ try {
+ testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ new File(testPath.toUri().getPath()).delete();
+ final Path finalTestPath = testPath;
+ CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+ @Override
+ public void printf(String fmt, Object... params) {
+ System.out.print(String.format(fmt, params));
+ }
+
+ public String readLine(String fmt, Object ... args) {
+ return finalTestPath.toString();
+ }
+
+ @Override
+ public char[] readPassword(String fmt, Object ... args) {
+ return defaultPass;
+ }
+ };
+
+ CredentialProviderUtility.main(new String[] {});
+
+ // now attempt to overwrite values
+ CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
+
+ int i = 0;
+
+ @Override
+ public void printf(String fmt, Object... params) {
+ System.out.print(String.format(fmt, params));
+ }
+
+ public String readLine(String fmt, Object ... args) {
+ return i++ == 0 ? finalTestPath.toString() : "y";
+ }
+
+ @Override
+ public char[] readPassword(String fmt, Object ... args) {
+ return new char[] {'n', 'e', 'w', 'p', 'a', 's', 's'};
+ }
+ };
+
+ CredentialProviderUtility.main(new String[] {});
+
+ String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
+ Configuration conf = new Configuration(false);
+
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ char[] newpass = "newpass".toCharArray();
+ CredentialProvider.CredentialEntry entry =
+ provider.getCredentialEntry(SecurityProperties.KEYSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry, newpass);
+ entry = provider.getCredentialEntry(SecurityProperties.TRUSTSTORE_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry, newpass);
+ entry = provider.getCredentialEntry(SecurityProperties.SERVER_CERT_PASSWORD_KEY);
+ assertCredentialEntryCorrect(entry, newpass);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
new file mode 100644
index 0000000..5782af7
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.filters;
+
+import org.apache.atlas.security.BaseSecurityTest;
+import org.apache.atlas.web.service.EmbeddedServer;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.mortbay.jetty.Server;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import javax.security.auth.Subject;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.PrivilegedExceptionAction;
+import java.util.Properties;
+
+/**
+ *
+ */
+public class MetadataAuthenticationKerberosFilterIT extends BaseSecurityTest {
+ public static final String TEST_USER_JAAS_SECTION = "TestUser";
+ public static final String TESTUSER = "testuser";
+ public static final String TESTPASS = "testpass";
+
+ private File userKeytabFile;
+ private File httpKeytabFile;
+
+ class TestEmbeddedServer extends EmbeddedServer {
+ public TestEmbeddedServer(int port, String path) throws IOException {
+ super(port, path);
+ }
+
+ Server getServer() {
+ return server;
+ }
+ }
+
+ @Test (enabled = false)
+ public void testKerberosBasedLogin() throws Exception {
+ String originalConf = System.getProperty("metadata.conf");
+ System.setProperty("metadata.conf", System.getProperty("user.dir"));
+
+ setupKDCAndPrincipals();
+ TestEmbeddedServer server = null;
+
+ try {
+ // setup the application.properties file
+ generateKerberosTestProperties();
+
+ // need to create the web application programmatically in order to control the injection of the test
+ // application properties
+ server = new TestEmbeddedServer(23000, "webapp/target/apache-atlas");
+
+ startEmbeddedServer(server.getServer());
+
+ final URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_SYSTEM_CONNECTION_FACTORY;
+ // attempt to hit server and get rejected
+ URL url = new URL("http://localhost:23000/");
+ HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, false);
+ connection.setRequestMethod("GET");
+ connection.connect();
+
+ Assert.assertEquals(connection.getResponseCode(), 401);
+
+ // need to populate the ticket cache with a local user, so logging in...
+ Subject subject = loginTestUser();
+
+ Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ // attempt to hit server and get rejected
+ URL url = new URL("http://localhost:23000/");
+ HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, true);
+ connection.setRequestMethod("GET");
+ connection.connect();
+
+ Assert.assertEquals(connection.getResponseCode(), 200);
+
+ return null;
+ }
+ });
+ } finally {
+ server.getServer().stop();
+ kdc.stop();
+
+ if (originalConf != null) {
+ System.setProperty("metadata.conf", originalConf);
+ } else {
+ System.clearProperty("metadata.conf");
+ }
+
+ }
+
+
+ }
+
+ protected Subject loginTestUser() throws LoginException, IOException {
+ LoginContext lc = new LoginContext(TEST_USER_JAAS_SECTION, new CallbackHandler() {
+
+ @Override
+ public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
+ for (int i = 0; i < callbacks.length; i++) {
+ if (callbacks[i] instanceof PasswordCallback) {
+ PasswordCallback passwordCallback = (PasswordCallback) callbacks[i];
+ passwordCallback.setPassword(TESTPASS.toCharArray());
+ }
+ if (callbacks[i] instanceof NameCallback) {
+ NameCallback nameCallback = (NameCallback) callbacks[i];
+ nameCallback.setName(TESTUSER);
+ }
+ }
+ }
+ });
+ // attempt authentication
+ lc.login();
+ return lc.getSubject();
+ }
+
+ protected void generateKerberosTestProperties() throws IOException, ConfigurationException {
+ Properties props = new Properties();
+ props.setProperty("atlas.http.authentication.enabled", "true");
+ props.setProperty("atlas.http.authentication.type", "kerberos");
+ props.setProperty("atlas.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
+ props.setProperty("atlas.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
+ props.setProperty("atlas.http.authentication.kerberos.name.rules",
+ "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
+
+ generateTestProperties(props);
+ }
+
+ public void setupKDCAndPrincipals() throws Exception {
+ // set up the KDC
+ File kdcWorkDir = startKDC();
+
+ userKeytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
+ httpKeytabFile = createKeytab(kdc, kdcWorkDir, "HTTP", "spnego.service.keytab");
+
+ // create a test user principal
+ kdc.createPrincipal(TESTUSER, TESTPASS);
+
+ StringBuilder jaas = new StringBuilder(1024);
+ jaas.append("TestUser {\n" +
+ " com.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true;\n" +
+ "};\n");
+ jaas.append(createJAASEntry("Client", "dgi", userKeytabFile));
+ jaas.append(createJAASEntry("Server", "HTTP", httpKeytabFile));
+
+ File jaasFile = new File(kdcWorkDir, "jaas.txt");
+ FileUtils.write(jaasFile, jaas.toString());
+ bindJVMtoJAASFile(jaasFile);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
new file mode 100644
index 0000000..aa6f0f9
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.filters;
+
+import org.apache.atlas.security.BaseSecurityTest;
+import org.apache.atlas.web.service.EmbeddedServer;
+import org.apache.commons.configuration.ConfigurationException;
+import org.mortbay.jetty.Server;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Properties;
+
+/**
+ *
+ */
+public class MetadataAuthenticationSimpleFilterIT extends BaseSecurityTest {
+
+ class TestEmbeddedServer extends EmbeddedServer {
+ public TestEmbeddedServer(int port, String path) throws IOException {
+ super(port, path);
+ }
+
+ Server getServer() {
+ return server;
+ }
+ }
+
+ @Test (enabled = false)
+ public void testSimpleLogin() throws Exception {
+ String originalConf = System.getProperty("metadata.conf");
+ System.setProperty("metadata.conf", System.getProperty("user.dir"));
+ generateSimpleLoginConfiguration();
+
+ TestEmbeddedServer server = new TestEmbeddedServer(23001, "webapp/target/apache-atlas");
+
+ try {
+ startEmbeddedServer(server.getServer());
+
+ URL url = new URL("http://localhost:23001");
+ HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+
+ try {
+ Assert.assertEquals(connection.getResponseCode(), 403);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ url = new URL("http://localhost:23001/?user.name=testuser");
+ connection = (HttpURLConnection)url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+
+ Assert.assertEquals(connection.getResponseCode(), 200);
+ } finally {
+ server.getServer().stop();
+ if (originalConf != null) {
+ System.setProperty("metadata.conf", originalConf);
+ } else {
+ System.clearProperty("metadata.conf");
+ }
+ }
+
+
+ }
+
+ protected void generateSimpleLoginConfiguration() throws IOException, ConfigurationException {
+ Properties config = new Properties();
+ config.setProperty("atlas.http.authentication.enabled", "true");
+ config.setProperty("atlas.http.authentication.type", "simple");
+
+ generateTestProperties(config);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
new file mode 100644
index 0000000..a250bca
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.listeners;
+
+import org.apache.atlas.security.BaseSecurityTest;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+
+/**
+ *
+ */
+public class LoginProcessorIT extends BaseSecurityTest {
+
+ protected static final String kerberosRule =
+ "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT";
+
+ @Test
+ public void testDefaultSimpleLogin() throws Exception {
+ LoginProcessor processor = new LoginProcessor() {
+ @Override
+ protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
+ return new PropertiesConfiguration();
+ }
+ };
+ processor.login();
+
+ Assert.assertNotNull(UserGroupInformation.getCurrentUser());
+ Assert.assertFalse(UserGroupInformation.isLoginKeytabBased());
+ Assert.assertFalse(UserGroupInformation.isSecurityEnabled());
+ }
+
+ @Test
+ public void testKerberosLogin() throws Exception {
+ final File keytab = setupKDCAndPrincipals();
+
+ LoginProcessor processor = new LoginProcessor() {
+ @Override
+ protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
+ PropertiesConfiguration config = new PropertiesConfiguration();
+ config.setProperty("atlas.authentication.method", "kerberos");
+ config.setProperty("atlas.authentication.principal", "dgi@EXAMPLE.COM");
+ config.setProperty("atlas.authentication.keytab", keytab.getAbsolutePath());
+ return config;
+ }
+
+ @Override
+ protected Configuration getHadoopConfiguration() {
+ Configuration config = new Configuration(false);
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ config.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, kerberosRule);
+
+ return config;
+ }
+
+ @Override
+ protected boolean isHadoopCluster() {
+ return true;
+ }
+ };
+ processor.login();
+
+ Assert.assertTrue(UserGroupInformation.getLoginUser().getShortUserName().endsWith("dgi"));
+ Assert.assertNotNull(UserGroupInformation.getCurrentUser());
+ Assert.assertTrue(UserGroupInformation.isLoginKeytabBased());
+ Assert.assertTrue(UserGroupInformation.isSecurityEnabled());
+
+ kdc.stop();
+
+ }
+
+ private File setupKDCAndPrincipals() throws Exception {
+ // set up the KDC
+ File kdcWorkDir = startKDC();
+
+ Assert.assertNotNull(kdc.getRealm());
+
+ File keytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
+
+ return keytabFile;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/AdminJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminJerseyResourceIT.java
new file mode 100755
index 0000000..a0410cb
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminJerseyResourceIT.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.atlas.web.util.Servlets;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+
+/**
+ * Integration test for Admin jersey resource.
+ */
+public class AdminJerseyResourceIT extends BaseResourceIT {
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Test
+ public void testGetVersion() throws Exception {
+ WebResource resource = service
+ .path("api/atlas/admin/version");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ PropertiesConfiguration buildConfiguration =
+ new PropertiesConfiguration("atlas-buildinfo.properties");
+
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertEquals(response.get("Version"),
+ buildConfiguration.getString("build.version"));
+ Assert.assertEquals(response.get("Name"),
+ buildConfiguration.getString("project.name"));
+ Assert.assertEquals(response.get("Description"),
+ buildConfiguration.getString("project.description"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
new file mode 100755
index 0000000..7c723e6
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
+
+/**
+ * Base class for integration tests.
+ * Sets up the web resource and has helper methods to create type and entity.
+ */
+public abstract class BaseResourceIT {
+
+ protected WebResource service;
+ protected MetadataServiceClient serviceClient;
+ public static String baseUrl = "http://localhost:21000/";
+
+ @BeforeClass
+ public void setUp() throws Exception {
+
+ DefaultClientConfig config = new DefaultClientConfig();
+ Client client = Client.create(config);
+ client.resource(UriBuilder.fromUri(baseUrl).build());
+
+ service = client.resource(UriBuilder.fromUri(baseUrl).build());
+ serviceClient = new MetadataServiceClient(baseUrl);
+ }
+
+ protected void createType(TypesDef typesDef) throws Exception {
+ HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0);
+ if (serviceClient.getType(sampleType.typeName) == null ) {
+ String typesAsJSON = TypesSerialization.toJson(typesDef);
+ createType(typesAsJSON);
+ }
+ }
+
+ protected void createType(String typesAsJSON) throws Exception {
+ WebResource resource = service
+ .path("api/atlas/types");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get("types"));
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+ }
+
+ protected Id createInstance(Referenceable referenceable) throws Exception {
+ String typeName = referenceable.getTypeName();
+ System.out.println("creating instance of type " + typeName);
+
+ String entityJSON = InstanceSerialization.toJson(referenceable, true);
+ System.out.println("Submitting new entity= " + entityJSON);
+ JSONObject jsonObject = serviceClient.createEntity(entityJSON);
+ String guid = jsonObject.getString(MetadataServiceClient.GUID);
+ System.out.println("created instance for type " + typeName + ", guid: " + guid);
+
+ // return the reference to created instance with guid
+ return new Id(guid, 0, referenceable.getTypeName());
+ }
+}
[10/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerialization.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerialization.scala b/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerialization.scala
deleted file mode 100755
index ae803c6..0000000
--- a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerialization.scala
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import java.text.SimpleDateFormat
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.TypesDef
-import org.apache.hadoop.metadata.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory}
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.MetadataException
-import org.json4s.JsonAST.JString
-import org.json4s._
-import org.json4s.native.Serialization._
-
-
-/**
- * Module for serializing to/from Json.
- *
- * @example {{{
- * val j = TypesSerialization.toJson(typeSystem, "Employee", "Person", "Department", "SecurityClearance")
- *
- * val typesDef = TypesSerialization.fromJson(jsonStr)
- * typesDef.enumTypes.foreach( typeSystem.defineEnumType(_))
-
- typeSystem.defineTypes(ImmutableList.copyOf(typesDef.structTypes.toArray),
- ImmutableList.copyOf(typesDef.traitTypes.toArray),
- ImmutableList.copyOf(typesDef.classTypes.toArray)
- )
- * }}}
- *
- * @todo doesn't traverse includes directives. Includes are parsed into
- * [[org.apache.hadoop.metadata.tools.thrift.IncludeDef IncludeDef]] structures
- * but are not traversed.
- * @todo mixing in [[scala.util.parsing.combinator.PackratParsers PackratParsers]] is a placeholder. Need to
- * change specific grammar rules to `lazy val` and `Parser[Elem]` to `PackratParser[Elem]`. Will do based on
- * performance analysis.
- * @todo Error reporting
- */
-object TypesSerialization {
-
- def toJsonValue(typ: IDataType[_])(implicit formats: Formats): JValue = {
- typ.getTypeCategory match {
- case TypeCategory.CLASS => {
- Extraction.decompose(convertClassTypeToHierarchicalTypeDefintion(typ.asInstanceOf[ClassType]))
- }
- case TypeCategory.STRUCT => {
- Extraction.decompose(convertStructTypeToStructDef(typ.asInstanceOf[StructType]))
- }
- case TypeCategory.TRAIT => {
- Extraction.decompose(convertTraitTypeToHierarchicalTypeDefintion(typ.asInstanceOf[TraitType]))
- }
- case TypeCategory.ENUM => {
- Extraction.decompose(convertEnumTypeToEnumTypeDef(typ.asInstanceOf[EnumType]))
- }
- case _ => JString(s"${typ.getName}")
- }
- }
-
- def toJson(ts: TypeSystem, typName: String): String = {
- toJson(ts, List(typName): _*)
- }
-
- def toJson(ts: TypeSystem, typNames: String*): String = {
- toJson(ts, (typ: IDataType[_]) => typNames.contains(typ.getName))
- }
-
- import scala.collection.JavaConversions._
-
- def toJson(ts: TypeSystem, typNames: java.util.List[String]): String = {
- toJson(ts, typNames.toIndexedSeq: _*)
- }
-
- val _formats = new DefaultFormats {
- override val dateFormatter = TypeSystem.getInstance().getDateFormat.asInstanceOf[SimpleDateFormat]
- override val typeHints = NoTypeHints
- }
-
- def toJson(ts: TypeSystem, export: IDataType[_] => Boolean): String = {
- implicit val formats = _formats + new MultiplicitySerializer
-
- val typsDef = convertToTypesDef(ts, export)
-
- writePretty(typsDef)
- }
-
- def fromJson(jsonStr: String): TypesDef = {
- implicit val formats = _formats + new MultiplicitySerializer
-
- read[TypesDef](jsonStr)
- }
-
- def toJson(typesDef : TypesDef) : String = {
- implicit val formats = _formats + new MultiplicitySerializer
- writePretty(typesDef)
-
- }
-
- def toJson(enumTypeDefinition: EnumTypeDefinition) : String = {
- toJson(new TypesDef(enumTypeDefinition))
- }
-
- def toJson(structTypeDefinition: StructTypeDefinition) : String = {
- toJson(new TypesDef(structTypeDefinition))
- }
-
- def toJson(typDef: HierarchicalTypeDefinition[_], isTrait : Boolean) : String = {
- toJson(new TypesDef(typDef, isTrait))
- }
-
- private def convertAttributeInfoToAttributeDef(aInfo: AttributeInfo) = {
- new AttributeDefinition(aInfo.name, aInfo.dataType().getName, aInfo.multiplicity,
- aInfo.isComposite, aInfo.isUnique, aInfo.isIndexable, aInfo.reverseAttributeName)
- }
-
- private def convertEnumTypeToEnumTypeDef(et: EnumType) = {
- val eVals: Seq[EnumValue] = et.valueMap.values().toSeq
- new EnumTypeDefinition(et.name, eVals: _*)
- }
-
- private def convertStructTypeToStructDef(st: StructType): StructTypeDefinition = {
-
- val aDefs: Iterable[AttributeDefinition] =
- st.fieldMapping.fields.values().map(convertAttributeInfoToAttributeDef(_))
- new StructTypeDefinition(st.name, aDefs.toArray)
- }
-
- private def convertTraitTypeToHierarchicalTypeDefintion(tt: TraitType): HierarchicalTypeDefinition[TraitType] = {
-
- val aDefs: Iterable[AttributeDefinition] =
- tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
- new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.superTypes, aDefs.toArray)
- }
-
- private def convertClassTypeToHierarchicalTypeDefintion(tt: ClassType): HierarchicalTypeDefinition[ClassType] = {
-
- val aDefs: Iterable[AttributeDefinition] =
- tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
- new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.superTypes, aDefs.toArray)
- }
-
- def convertToTypesDef(ts: TypeSystem, export: IDataType[_] => Boolean): TypesDef = {
-
- var enumTypes: Seq[EnumTypeDefinition] = Nil
- var structTypes: Seq[StructTypeDefinition] = Nil
- var traitTypes: Seq[HierarchicalTypeDefinition[TraitType]] = Nil
- var classTypes: Seq[HierarchicalTypeDefinition[ClassType]] = Nil
-
- def toTyp(nm: String) = ts.getDataType(classOf[IDataType[_]], nm)
-
- val typs: Iterable[IDataType[_]] = ts.getTypeNames.map(toTyp(_)).filter { (typ: IDataType[_]) =>
- !(ts.getCoreTypes.contains(typ.getName)) && export(typ)
- }
-
- typs.foreach {
- case typ: ArrayType => ()
- case typ: MapType => ()
- case typ: EnumType => enumTypes = enumTypes :+ convertEnumTypeToEnumTypeDef(typ)
- case typ: StructType => structTypes = structTypes :+ convertStructTypeToStructDef(typ)
- case typ: TraitType => traitTypes = traitTypes :+ convertTraitTypeToHierarchicalTypeDefintion(typ)
- case typ: ClassType => classTypes = classTypes :+ convertClassTypeToHierarchicalTypeDefintion(typ)
- }
-
- TypesDef(enumTypes, structTypes, traitTypes, classTypes)
- }
-
-}
-
-class MultiplicitySerializer extends CustomSerializer[Multiplicity](format => ( {
- case JString(m) => m match {
- case "optional" => Multiplicity.OPTIONAL
- case "required" => Multiplicity.REQUIRED
- case "collection" => Multiplicity.COLLECTION
- case "set" => Multiplicity.SET
- }
-}, {
- case m: Multiplicity => JString(m match {
- case Multiplicity.OPTIONAL => "optional"
- case Multiplicity.REQUIRED => "required"
- case Multiplicity.COLLECTION => "collection"
- case Multiplicity.SET => "set"
- }
-
- )
-}
- ))
-
-trait TypeHelpers {
- def requiredAttr(name: String, dataType: IDataType[_]) =
- new AttributeDefinition(name, dataType.getName, Multiplicity.REQUIRED, false, null)
-
- def optionalAttr(name: String, dataTypeName: String) =
- new AttributeDefinition(name, dataTypeName, Multiplicity.OPTIONAL, false, null)
-
-
- def optionalAttr(name: String, dataType: IDataType[_]) =
- new AttributeDefinition(name, dataType.getName, Multiplicity.OPTIONAL, false, null)
-
- def structDef(name: String, attrs: AttributeDefinition*) = {
- new StructTypeDefinition(name, attrs.toArray)
- }
-
- def defineTraits(ts: TypeSystem, tDefs: HierarchicalTypeDefinition[TraitType]*) = {
- ts.defineTraitTypes(tDefs: _*)
- }
-
- def createTraitTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*):
- HierarchicalTypeDefinition[TraitType] = {
- val sts = ImmutableList.copyOf(superTypes.toArray)
- return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name,
- sts, attrDefs.toArray)
- }
-
- def createClassTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*):
- HierarchicalTypeDefinition[ClassType] = {
- val sts = ImmutableList.copyOf(superTypes.toArray)
- return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name,
- sts, attrDefs.toArray)
- }
-
- @throws(classOf[MetadataException])
- def defineClassType(ts: TypeSystem, classDef: HierarchicalTypeDefinition[ClassType]): ClassType = {
- ts.defineTypes(ImmutableList.of[StructTypeDefinition],
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]](classDef))
- return ts.getDataType(classOf[ClassType], classDef.typeName)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java
new file mode 100755
index 0000000..dde66f8
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/json/SerializationJavaTest.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.BaseTest;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
+
+public class SerializationJavaTest extends BaseTest {
+
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ /*
+ * Class Hierarchy is:
+ * Department(name : String, employees : Array[Person])
+ * Person(name : String, department : Department, manager : Manager)
+ * Manager(subordinates : Array[Person]) extends Person
+ *
+ * Persons can have SecurityClearance(level : Int) clearance.
+ */
+ @Test
+ public void test1() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+
+ HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department",
+ ImmutableList.<String>of(),
+ createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees",
+ String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true,
+ "department")
+ );
+ HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person",
+ ImmutableList.<String>of(),
+ createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department",
+ "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager",
+ "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ );
+ HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
+ ImmutableList.<String>of("Person"),
+ new AttributeDefinition("subordinates",
+ String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false,
+ "manager")
+ );
+
+ HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef(
+ "SecurityClearance",
+ ImmutableList.<String>of(),
+ createRequiredAttrDef("level", DataTypes.INT_TYPE)
+ );
+
+ ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef,
+ managerTypeDef));
+
+ Referenceable hrDept = new Referenceable("Department");
+ Referenceable john = new Referenceable("Person");
+ Referenceable jane = new Referenceable("Manager", "SecurityClearance");
+
+ hrDept.set("name", "hr");
+ john.set("name", "John");
+ john.set("department", hrDept);
+ jane.set("name", "Jane");
+ jane.set("department", hrDept);
+
+ john.set("manager", jane);
+
+ hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
+
+ jane.set("subordinates", ImmutableList.<Referenceable>of(john));
+
+ jane.getTrait("SecurityClearance").set("level", 1);
+
+ ClassType deptType = ts.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+ String hrDeptStr = hrDept2.toString();
+
+ Assert.assertEquals(hrDeptStr, "{\n" +
+ "\tid : (type: Department, id: <unassigned>)\n" +
+ "\tname : \thr\n" +
+ "\temployees : \t[{\n" +
+ "\tid : (type: Person, id: <unassigned>)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : (type: Manager, id: <unassigned>)\n" +
+ "}, {\n" +
+ "\tid : (type: Manager, id: <unassigned>)\n" +
+ "\tsubordinates : \t[{\n" +
+ "\tid : (type: Person, id: <unassigned>)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : (type: Manager, id: <unassigned>)\n" +
+ "}]\n" +
+ "\tname : \tJane\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : <null>\n" +
+ "\n" +
+ "\tSecurityClearance : \t{\n" +
+ "\t\tlevel : \t\t1\n" +
+ "\t}}]\n" +
+ "}");
+
+ String jsonStr = Serialization$.MODULE$.toJson(hrDept2);
+ //System.out.println(jsonStr);
+
+ hrDept2 = Serialization$.MODULE$.fromJson(jsonStr);
+ Assert.assertEquals(hrDept2.toString(), hrDeptStr);
+
+ }
+
+ @Test
+ public void testTrait() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+ HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef(
+ "SecurityClearance2",
+ ImmutableList.<String>of(),
+ createRequiredAttrDef("level", DataTypes.INT_TYPE)
+ );
+
+ ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+
+
+ Struct s = new Struct("SecurityClearance2");
+ s.set("level", 1);
+ TraitType tType = ts.getDataType(TraitType.class, "SecurityClearance2");
+ ITypedInstance t = tType.convert(s, Multiplicity.REQUIRED);
+ String jsonStr = Serialization$.MODULE$.toJson(t);
+ ITypedInstance t2 = Serialization$.MODULE$.traitFromJson(jsonStr);
+ Assert.assertEquals(t.toString(), t2.toString());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
new file mode 100755
index 0000000..f5c6f7f
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.junit.Before;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.Map;
+
+public abstract class BaseTest {
+
+ public static final String STRUCT_TYPE_1 = "t1";
+ public static final String STRUCT_TYPE_2 = "t2";
+ public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
+ public static final long TEST_DATE_IN_LONG=1418265358440L;
+
+ public static Struct createStruct() throws MetadataException {
+ StructType structType = TypeSystem.getInstance().getDataType(
+ StructType.class, STRUCT_TYPE_1);
+ Struct s = new Struct(structType.getName());
+ s.set("a", 1);
+ s.set("b", true);
+ s.set("c", (byte) 1);
+ s.set("d", (short) 2);
+ s.set("e", 1);
+ s.set("f", 1);
+ s.set("g", 1L);
+ s.set("h", 1.0f);
+ s.set("i", 1.0);
+ s.set("j", BigInteger.valueOf(1L));
+ s.set("k", new BigDecimal(1));
+ s.set("l", new Date(1418265358440L));
+ s.set("m", Lists.asList(1, new Integer[]{1}));
+ s.set("n", Lists.asList(BigDecimal.valueOf(1.1),
+ new BigDecimal[]{BigDecimal.valueOf(1.1)}));
+ Map<String, Double> hm = Maps.newHashMap();
+ hm.put("a", 1.0);
+ hm.put("b", 2.0);
+ s.set("o", hm);
+ return s;
+ }
+
+ protected final TypeSystem getTypeSystem() {
+ return TypeSystem.getInstance();
+ }
+
+ @Before
+ public void setup() throws Exception {
+ TypeSystem ts = TypeSystem.getInstance();
+ ts.reset();
+
+ StructType structType = ts.defineStructType(STRUCT_TYPE_1,
+ true,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+ TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+ TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+ TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+ TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+ TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+ TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+ TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ TypesUtil.createOptionalAttrDef("o",
+ ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
+ System.out.println("defined structType = " + structType);
+
+ StructType recursiveStructType = ts.defineStructType(STRUCT_TYPE_2,
+ true,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
+ System.out.println("defined recursiveStructType = " + recursiveStructType);
+ }
+
+ protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs)
+ throws MetadataException {
+
+ return getTypeSystem().defineTraitTypes(tDefs);
+ }
+
+ protected Map<String, IDataType> defineClasses(
+ HierarchicalTypeDefinition<ClassType>... classDefs) throws MetadataException {
+ return getTypeSystem().defineClassTypes(classDefs);
+ }
+
+ /*
+ * Class Hierarchy is:
+ * Department(name : String, employees : Array[Person])
+ * Person(name : String, department : Department, manager : Manager)
+ * Manager(subordinates : Array[Person]) extends Person
+ *
+ * Persons can have SecurityClearance(level : Int) clearance.
+ */
+ protected void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
+
+ HierarchicalTypeDefinition<ClassType> deptTypeDef = TypesUtil
+ .createClassTypeDef("Department",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees",
+ String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true,
+ "department")
+ );
+ HierarchicalTypeDefinition<ClassType> personTypeDef = TypesUtil.createClassTypeDef("Person",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department",
+ "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager",
+ "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ );
+ HierarchicalTypeDefinition<ClassType> managerTypeDef =
+ TypesUtil.createClassTypeDef("Manager",
+ ImmutableList.of("Person"),
+ new AttributeDefinition("subordinates",
+ String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, false, "manager")
+ );
+
+ HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef =
+ TypesUtil.createTraitTypeDef(
+ "SecurityClearance",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE)
+ );
+
+ ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(securityClearanceTypeDef),
+ ImmutableList.of(deptTypeDef, personTypeDef,
+ managerTypeDef));
+
+ ImmutableList.of(
+ ts.getDataType(HierarchicalType.class, "SecurityClearance"),
+ ts.getDataType(ClassType.class, "Department"),
+ ts.getDataType(ClassType.class, "Person"),
+ ts.getDataType(ClassType.class, "Manager")
+ );
+ }
+
+ protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
+ Referenceable hrDept = new Referenceable("Department");
+ Referenceable john = new Referenceable("Person");
+ Referenceable jane = new Referenceable("Manager", "SecurityClearance");
+
+ hrDept.set("name", "hr");
+ john.set("name", "John");
+ john.set("department", hrDept);
+ jane.set("name", "Jane");
+ jane.set("department", hrDept);
+
+ john.set("manager", jane);
+
+ hrDept.set("employees", ImmutableList.of(john, jane));
+
+ jane.set("subordinates", ImmutableList.of(john));
+
+ jane.getTrait("SecurityClearance").set("level", 1);
+
+ ClassType deptType = ts.getDataType(ClassType.class, "Department");
+ deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+ return hrDept;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
new file mode 100755
index 0000000..c7bb448
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ClassTest extends BaseTest {
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @Test
+ public void test1() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+
+ defineDeptEmployeeTypes(ts);
+ Referenceable hrDept = createDeptEg1(ts);
+ ClassType deptType = ts.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+
+ Assert.assertEquals(hrDept2.toString(), "{\n" +
+ "\tid : (type: Department, id: <unassigned>)\n" +
+ "\tname : \thr\n" +
+ "\temployees : \t[{\n" +
+ "\tid : (type: Person, id: <unassigned>)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : (type: Manager, id: <unassigned>)\n" +
+ "}, {\n" +
+ "\tid : (type: Manager, id: <unassigned>)\n" +
+ "\tsubordinates : \t[{\n" +
+ "\tid : (type: Person, id: <unassigned>)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : (type: Manager, id: <unassigned>)\n" +
+ "}]\n" +
+ "\tname : \tJane\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : <null>\n" +
+ "\n" +
+ "\tSecurityClearance : \t{\n" +
+ "\t\tlevel : \t\t1\n" +
+ "\t}}]\n" +
+ "}");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
new file mode 100755
index 0000000..4677d07
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.Map;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+
+public class EnumTest extends BaseTest {
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ void defineEnums(TypeSystem ts) throws MetadataException {
+ ts.defineEnumType("HiveObjectType",
+ new EnumValue("GLOBAL", 1),
+ new EnumValue("DATABASE", 2),
+ new EnumValue("TABLE", 3),
+ new EnumValue("PARTITION", 4),
+ new EnumValue("COLUMN", 5));
+
+ ts.defineEnumType("PrincipalType",
+ new EnumValue("USER", 1),
+ new EnumValue("ROLE", 2),
+ new EnumValue("GROUP", 3));
+
+ ts.defineEnumType("TxnState",
+ new EnumValue("COMMITTED", 1),
+ new EnumValue("ABORTED", 2),
+ new EnumValue("OPEN", 3));
+
+ ts.defineEnumType("LockLevel",
+ new EnumValue("DB", 1),
+ new EnumValue("TABLE", 2),
+ new EnumValue("PARTITION", 3));
+
+ }
+
+ protected void fillStruct(Struct s) throws MetadataException {
+ s.set("a", 1);
+ s.set("b", true);
+ s.set("c", (byte) 1);
+ s.set("d", (short) 2);
+ s.set("e", 1);
+ s.set("f", 1);
+ s.set("g", 1L);
+ s.set("h", 1.0f);
+ s.set("i", 1.0);
+ s.set("j", BigInteger.valueOf(1L));
+ s.set("k", new BigDecimal(1));
+ s.set("l", new Date(1418265358440L));
+ s.set("m", Lists.asList(1, new Integer[]{1}));
+ s.set("n",
+ Lists.asList(BigDecimal.valueOf(1.1), new BigDecimal[]{BigDecimal.valueOf(1.1)}));
+ Map<String, Double> hm = Maps.newHashMap();
+ hm.put("a", 1.0);
+ hm.put("b", 2.0);
+ s.set("o", hm);
+ s.set("enum1", "GLOBAL");
+ s.set("enum2", 1);
+ s.set("enum3", "COMMITTED");
+ s.set("enum4", 3);
+ }
+
+ protected Struct createStructWithEnum(String typeName) throws MetadataException {
+ Struct s = new Struct(typeName);
+ fillStruct(s);
+ return s;
+ }
+
+ protected Referenceable createInstanceWithEnum(String typeName) throws MetadataException {
+ Referenceable r = new Referenceable(typeName);
+ fillStruct(r);
+ return r;
+ }
+
+ protected ClassType defineClassTypeWithEnum(TypeSystem ts) throws MetadataException {
+ return ts.defineClassType(createClassTypeDef("t4",
+ ImmutableList.<String>of(),
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+ createOptionalAttrDef("enum1", ts.getDataType(EnumType.class, "HiveObjectType")),
+ createOptionalAttrDef("e", DataTypes.INT_TYPE),
+ createOptionalAttrDef("f", DataTypes.INT_TYPE),
+ createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+ createOptionalAttrDef("enum2", ts.getDataType(EnumType.class, "PrincipalType")),
+ createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+ createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+ createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+ createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+ createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
+ createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+ createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ createOptionalAttrDef("o",
+ ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ createOptionalAttrDef("enum4", ts.getDataType(EnumType.class, "LockLevel"))));
+ }
+
+ @Test
+ public void testStruct() throws MetadataException {
+ TypeSystem ts = getTypeSystem();
+ defineEnums(ts);
+ StructType structType = ts.defineStructType("t3",
+ true,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+ createOptionalAttrDef("enum1", ts.getDataType(EnumType.class, "HiveObjectType")),
+ createOptionalAttrDef("e", DataTypes.INT_TYPE),
+ createOptionalAttrDef("f", DataTypes.INT_TYPE),
+ createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+ createOptionalAttrDef("enum2", ts.getDataType(EnumType.class, "PrincipalType")),
+ createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+ createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+ createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+ createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+ createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
+ createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+ createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ createOptionalAttrDef("o",
+ ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ createOptionalAttrDef("enum4", ts.getDataType(EnumType.class, "LockLevel")));
+
+ Struct s = createStructWithEnum("t3");
+ ITypedStruct typedS = structType.convert(s, Multiplicity.REQUIRED);
+ Assert.assertEquals(typedS.toString(), "{\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t2\n" +
+ "\tenum1 : \tGLOBAL\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\tenum2 : \tUSER\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tenum3 : \tCOMMITTED\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.1, 1.1]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "\tenum4 : \tPARTITION\n" +
+ "}");
+ }
+
+ @Test
+ public void testClass() throws MetadataException {
+ TypeSystem ts = getTypeSystem();
+ defineEnums(ts);
+ ClassType clsType = defineClassTypeWithEnum(ts);
+
+ IReferenceableInstance r = createInstanceWithEnum("t4");
+ ITypedReferenceableInstance typedR = clsType.convert(r, Multiplicity.REQUIRED);
+ Assert.assertEquals(typedR.toString(), "{\n" +
+ "\tid : (type: t4, id: <unassigned>)\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t2\n" +
+ "\tenum1 : \tGLOBAL\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\tenum2 : \tUSER\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tenum3 : \tCOMMITTED\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.1, 1.1]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "\tenum4 : \tPARTITION\n" +
+ "}");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
new file mode 100755
index 0000000..e063bed
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Struct;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StructTest extends BaseTest {
+
+ StructType structType;
+ StructType recursiveStructType;
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ structType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
+ recursiveStructType = getTypeSystem()
+ .getDataType(StructType.class, STRUCT_TYPE_2);
+ }
+
+ @Test
+ public void test1() throws MetadataException {
+ Struct s = createStruct();
+ ITypedStruct ts = structType.convert(s, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t2\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.1, 1.1]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "}");
+ }
+
+ @Test
+ public void testRecursive() throws MetadataException {
+ Struct s1 = new Struct(recursiveStructType.getName());
+ s1.set("a", 1);
+ Struct s2 = new Struct(recursiveStructType.getName());
+ s2.set("a", 1);
+ s2.set("s", s1);
+ ITypedStruct ts = recursiveStructType.convert(s2, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\ta : \t1\n" +
+ "\ts : \t{\n" +
+ "\t\ta : \t\t1\n" +
+ "\t\ts : <null>\n" +
+ "\n" +
+ "\t}\n" +
+ "}");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
new file mode 100755
index 0000000..31a1128
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Struct;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
+
+public class TraitTest extends BaseTest {
+
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ /*
+ * Type Hierarchy is:
+ * A(a,b,c,d)
+ * B(b) extends A
+ * C(c) extends A
+ * D(d) extends B,C
+ *
+ * - There are a total of 11 fields in an instance of D
+ * - an attribute that is hidden by a SubType can referenced by prefixing it with the
+ * complete Path.
+ * For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
+ * in B.
+ * So it is availabel by the name 'A.B.D.b'
+ *
+ * - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
+ * .b' attribute
+ * available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
+ * available as 'b'.
+ */
+ @Test
+ public void test1() throws MetadataException {
+ HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+ HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+ HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
+ HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+
+ defineTraits(A, B, C, D);
+
+ TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
+
+// for(String aName : DType.fieldMapping().fields.keySet()) {
+// System.out.println(String.format("nameToQualifiedName.put(\"%s\", \"%s\");", aName, DType.getQualifiedName(aName)));
+// }
+
+ Map<String,String> nameToQualifiedName = new HashMap();
+ {
+ nameToQualifiedName.put("d", "D.d");
+ nameToQualifiedName.put("b", "B.b");
+ nameToQualifiedName.put("c", "C.c");
+ nameToQualifiedName.put("a", "A.a");
+ nameToQualifiedName.put("A.B.D.b", "A.B.D.b");
+ nameToQualifiedName.put("A.B.D.c", "A.B.D.c");
+ nameToQualifiedName.put("A.B.D.d", "A.B.D.d");
+ nameToQualifiedName.put("A.C.D.a", "A.C.D.a");
+ nameToQualifiedName.put("A.C.D.b", "A.C.D.b");
+ nameToQualifiedName.put("A.C.D.c", "A.C.D.c");
+ nameToQualifiedName.put("A.C.D.d", "A.C.D.d");
+ }
+
+ Struct s1 = new Struct("D");
+ s1.set("d", 1);
+ s1.set("c", 1);
+ s1.set("b", true);
+ s1.set("a", 1);
+ s1.set("A.B.D.b", true);
+ s1.set("A.B.D.c", 2);
+ s1.set("A.B.D.d", 2);
+
+ s1.set("A.C.D.a", 3);
+ s1.set("A.C.D.b", false);
+ s1.set("A.C.D.c", 3);
+ s1.set("A.C.D.d", 3);
+
+
+ ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ /*
+ * cast to B and set the 'b' attribute on A.
+ */
+ TraitType BType = (TraitType) getTypeSystem().getDataType(TraitType.class, "B");
+ IStruct s2 = DType.castAs(ts, "B");
+ s2.set("A.B.b", false);
+
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \tfalse\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ /*
+ * cast again to A and set the 'b' attribute on A.
+ */
+ TraitType AType = (TraitType) getTypeSystem().getDataType(TraitType.class, "A");
+ IStruct s3 = BType.castAs(s2, "A");
+ s3.set("b", true);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+ }
+
+ @Test
+ public void testRandomOrder() throws MetadataException {
+ HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+ HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+ HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
+ HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+
+ defineTraits(B, D, A, C);
+
+ TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
+
+ Struct s1 = new Struct("D");
+ s1.set("d", 1);
+ s1.set("c", 1);
+ s1.set("b", true);
+ s1.set("a", 1);
+ s1.set("A.B.D.b", true);
+ s1.set("A.B.D.c", 2);
+ s1.set("A.B.D.d", 2);
+
+ s1.set("A.C.D.a", 3);
+ s1.set("A.C.D.b", false);
+ s1.set("A.C.D.c", 3);
+ s1.set("A.C.D.d", 3);
+
+
+ ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeInheritanceTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeInheritanceTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeInheritanceTest.java
new file mode 100644
index 0000000..8829862
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeInheritanceTest.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Struct;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
+
+/**
+ * Unit tests for type inheritance.
+ */
+public class TypeInheritanceTest extends BaseTest {
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ TypeSystem.getInstance().reset();
+ super.setup();
+ }
+
+ /*
+ * Type Hierarchy is:
+ * A(a)
+ * B(b) extends A
+ */
+ @Test
+ public void testSimpleInheritance() throws MetadataException {
+ HierarchicalTypeDefinition A = createClassTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE));
+
+ HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+
+ defineClasses(A, B);
+
+ ClassType BType = getTypeSystem().getDataType(ClassType.class, "B");
+
+ Struct s1 = new Struct("B");
+ s1.set("b", true);
+ s1.set("a", 1);
+
+ ITypedInstance ts = BType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\tid : (type: B, id: <unassigned>)\n" +
+ "\tb : \ttrue\n" +
+ "\ta : \t1\n" +
+ "}");
+ }
+
+ /*
+ * Type Hierarchy is:
+ * A(a, b)
+ * B(b) extends A
+ */
+ @Test
+ public void testSimpleInheritanceWithOverrides() throws MetadataException {
+ HierarchicalTypeDefinition A = createClassTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createRequiredAttrDef("b", DataTypes.BOOLEAN_TYPE));
+
+ HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+
+ defineClasses(A, B);
+
+ ClassType BType = getTypeSystem().getDataType(ClassType.class, "B");
+
+ Struct s1 = new Struct("B");
+ s1.set("b", true);
+ s1.set("a", 1);
+ s1.set("A.B.b", false);
+
+ ITypedInstance ts = BType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\tid : (type: B, id: <unassigned>)\n" +
+ "\tb : \ttrue\n" +
+ "\ta : \t1\n" +
+ "\tA.B.b : \tfalse\n" +
+ "}");
+ }
+
+ /*
+ * Type Hierarchy is:
+ * A(a)
+ * B(b) extends A
+ * C(c) extends B
+ * D(d) extends C
+ */
+ @Test
+ public void testMultiLevelInheritance() throws MetadataException {
+ HierarchicalTypeDefinition A = createClassTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE));
+
+ HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+
+ HierarchicalTypeDefinition C = createClassTypeDef("C", ImmutableList.of("B"),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
+
+ HierarchicalTypeDefinition D = createClassTypeDef("D", ImmutableList.of("C"),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+
+ defineClasses(A, B, C, D);
+
+ ClassType DType = getTypeSystem().getDataType(ClassType.class, "D");
+
+ Struct s1 = new Struct("D");
+ s1.set("d", 1);
+ s1.set("c", 1);
+ s1.set("b", true);
+ s1.set("a", 1);
+
+ ITypedInstance ts = DType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\tid : (type: D, id: <unassigned>)\n" +
+ "\td : \t1\n" +
+ "\tc : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\ta : \t1\n" +
+ "}");
+ }
+
+ /*
+ * Type Hierarchy is:
+ * A(a,b,c,d)
+ * B(b) extends A
+ * C(c) extends A
+ * D(d) extends B,C
+ *
+ * - There are a total of 11 fields in an instance of D
+ * - an attribute that is hidden by a SubType can referenced by prefixing it with the
+ * complete Path.
+ * For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
+ * in B.
+ * So it is availabel by the name 'A.B.D.b'
+ *
+ * - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
+ * .b' attribute
+ * available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
+ * available as 'b'.
+ */
+ @Test
+ public void testDiamondInheritance() throws MetadataException {
+ HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+ HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+ HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.of("A"),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
+ HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.of("B", "C"),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+
+ defineTraits(A, B, C, D);
+
+ TraitType DType = getTypeSystem().getDataType(TraitType.class, "D");
+
+ Struct s1 = new Struct("D");
+ s1.set("d", 1);
+ s1.set("c", 1);
+ s1.set("b", true);
+ s1.set("a", 1);
+ s1.set("A.B.D.b", true);
+ s1.set("A.B.D.c", 2);
+ s1.set("A.B.D.d", 2);
+
+ s1.set("A.C.D.a", 3);
+ s1.set("A.C.D.b", false);
+ s1.set("A.C.D.c", 3);
+ s1.set("A.C.D.d", 3);
+
+
+ ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ /*
+ * cast to B and set the 'b' attribute on A.
+ */
+ TraitType BType = getTypeSystem().getDataType(TraitType.class, "B");
+ IStruct s2 = DType.castAs(ts, "B");
+ s2.set("A.B.b", false);
+
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \tfalse\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ /*
+ * cast again to A and set the 'b' attribute on A.
+ */
+ IStruct s3 = BType.castAs(s2, "A");
+ s3.set("b", true);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java
new file mode 100755
index 0000000..4f43ad0
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TypeSystemTest.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+import scala.actors.threadpool.Arrays;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
+
+public class TypeSystemTest extends BaseTest {
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ super.setup();
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ getTypeSystem().reset();
+ }
+
+ @Test
+ public void testGetTypeNames() throws Exception {
+ getTypeSystem().defineEnumType("enum_test",
+ new EnumValue("0", 0),
+ new EnumValue("1", 1),
+ new EnumValue("2", 2),
+ new EnumValue("3", 3));
+ Assert.assertTrue(getTypeSystem().getTypeNames().contains("enum_test"));
+ }
+
+ @Test
+ public void testIsRegistered() throws Exception {
+ getTypeSystem().defineEnumType("enum_test",
+ new EnumValue("0", 0),
+ new EnumValue("1", 1),
+ new EnumValue("2", 2),
+ new EnumValue("3", 3));
+ Assert.assertTrue(getTypeSystem().isRegistered("enum_test"));
+ }
+
+ @Test
+ public void testGetTraitsNames() throws Exception {
+ HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
+ TypesUtil.createTraitTypeDef("Classification",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef("PII", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> phiTrait =
+ TypesUtil.createTraitTypeDef("PHI", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> pciTrait =
+ TypesUtil.createTraitTypeDef("PCI", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> soxTrait =
+ TypesUtil.createTraitTypeDef("SOX", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> secTrait =
+ TypesUtil.createTraitTypeDef("SEC", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> financeTrait =
+ TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
+
+ getTypeSystem().defineTypes(
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
+ soxTrait, secTrait, financeTrait),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+
+ final ImmutableList<String> traitsNames = getTypeSystem().getTypeNamesByCategory(DataTypes.TypeCategory.TRAIT);
+ Assert.assertEquals(traitsNames.size(), 7);
+ List traits = Arrays.asList(new String[]{
+ "Classification",
+ "PII",
+ "PHI",
+ "PCI",
+ "SOX",
+ "SEC",
+ "Finance",
+ });
+
+ Assert.assertFalse(Collections.disjoint(traitsNames, traits));
+ }
+
+ private String random() {
+ return RandomStringUtils.random(10);
+ }
+
+ @Test
+ public void testUTFNames() throws Exception {
+ TypeSystem ts = getTypeSystem();
+
+ String enumType = random();
+ EnumTypeDefinition orgLevelEnum =
+ new EnumTypeDefinition(enumType, new EnumValue(random(), 1), new EnumValue(random(), 2));
+ ts.defineEnumType(orgLevelEnum);
+
+ String structName = random();
+ String attrType = random();
+ StructTypeDefinition structType = createStructTypeDef(structName,
+ createRequiredAttrDef(attrType, DataTypes.STRING_TYPE));
+
+ String className = random();
+ HierarchicalTypeDefinition<ClassType> classType =
+ createClassTypeDef(className, ImmutableList.<String>of(),
+ createRequiredAttrDef(attrType, DataTypes.STRING_TYPE));
+
+ String traitName = random();
+ HierarchicalTypeDefinition<TraitType> traitType = createTraitTypeDef(traitName,
+ ImmutableList.<String>of(), createRequiredAttrDef(attrType, DataTypes.INT_TYPE));
+
+ ts.defineTypes(ImmutableList.of(structType), ImmutableList.of(traitType), ImmutableList.of(classType));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/atlas/typesystem/types/ValidationTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ValidationTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ValidationTest.java
new file mode 100644
index 0000000..29638de
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ValidationTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class ValidationTest {
+ @DataProvider(name = "attributeData")
+ private Object[][] createAttributeData() {
+ return new String[][]{
+ {null, "type"}, {"", "type"}, {"name", null}, {"name", ""}};
+ }
+
+ @Test (dataProvider = "attributeData", expectedExceptions = {IllegalArgumentException.class})
+ public void testAttributes(String name, String type) {
+ TypesUtil.createRequiredAttrDef(name, type);
+ }
+
+ @DataProvider(name = "enumValueData")
+ private Object[][] createEnumValueData() {
+ return new String[][]{{null}, {""}};
+ }
+
+ @Test (dataProvider = "enumValueData", expectedExceptions = {IllegalArgumentException.class})
+ public void testEnumValue(String name) {
+ new EnumValue(name, 1);
+ }
+
+ @DataProvider(name = "enumTypeData")
+ private Object[][] createEnumTypeData() {
+ EnumValue value = new EnumValue("name", 1);
+ return new Object[][]{{null, value}, {"", value}, {"name"}};
+ }
+
+ @Test (dataProvider = "enumTypeData", expectedExceptions = {IllegalArgumentException.class})
+ public void testEnumType(String name, EnumValue... values) {
+ new EnumTypeDefinition(name, values);
+ }
+
+ @DataProvider(name = "structTypeData")
+ private Object[][] createStructTypeData() {
+ AttributeDefinition value = TypesUtil.createRequiredAttrDef("name", "type");
+ return new Object[][]{{null, value}, {"", value}, {"name"}};
+ }
+
+ @Test (dataProvider = "structTypeData", expectedExceptions = {IllegalArgumentException.class})
+ public void testStructType(String name, AttributeDefinition... values) {
+ new StructTypeDefinition(name, values);
+ }
+
+ @DataProvider(name = "classTypeData")
+ private Object[][] createClassTypeData() {
+ return new Object[][]{{null}, {""}};
+ }
+
+ @Test (dataProvider = "classTypeData", expectedExceptions = {IllegalArgumentException.class})
+ public void testClassType(String name) {
+ AttributeDefinition value = TypesUtil.createRequiredAttrDef("name", "type");;
+ TypesUtil.createClassTypeDef(name, ImmutableList.of("super"), value);
+ }
+
+ @Test (dataProvider = "classTypeData", expectedExceptions = {IllegalArgumentException.class})
+ public void testTraitType(String name) {
+ AttributeDefinition value = TypesUtil.createRequiredAttrDef("name", "type");;
+ TypesUtil.createTraitTypeDef(name, ImmutableList.of("super"), value);
+ }
+
+ @Test
+ public void testValidTypes() {
+ AttributeDefinition attribute = TypesUtil.createRequiredAttrDef("name", "type");
+
+ //class with no attributes
+ TypesUtil.createClassTypeDef("name", ImmutableList.of("super"));
+
+ //class with no super types
+ TypesUtil.createClassTypeDef("name", ImmutableList.<String>of(), attribute);
+
+ //trait with no attributes
+ TypesUtil.createTraitTypeDef("name", ImmutableList.of("super"));
+
+ //trait with no super types
+ TypesUtil.createTraitTypeDef("name", ImmutableList.<String>of(), attribute);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/json/SerializationJavaTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/json/SerializationJavaTest.java b/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/json/SerializationJavaTest.java
deleted file mode 100755
index c2ef616..0000000
--- a/typesystem/src/test/java/org/apache/hadoop/metadata/typesystem/json/SerializationJavaTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.ITypedInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.BaseTest;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-
-public class SerializationJavaTest extends BaseTest {
-
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- /*
- * Class Hierarchy is:
- * Department(name : String, employees : Array[Person])
- * Person(name : String, department : Department, manager : Manager)
- * Manager(subordinates : Array[Person]) extends Person
- *
- * Persons can have SecurityClearance(level : Int) clearance.
- */
- @Test
- public void test1() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
-
- HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department",
- ImmutableList.<String>of(),
- createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees",
- String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true,
- "department")
- );
- HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person",
- ImmutableList.<String>of(),
- createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department",
- "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager",
- "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- );
- HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
- ImmutableList.<String>of("Person"),
- new AttributeDefinition("subordinates",
- String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false,
- "manager")
- );
-
- HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef(
- "SecurityClearance",
- ImmutableList.<String>of(),
- createRequiredAttrDef("level", DataTypes.INT_TYPE)
- );
-
- ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
- ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef,
- managerTypeDef));
-
- Referenceable hrDept = new Referenceable("Department");
- Referenceable john = new Referenceable("Person");
- Referenceable jane = new Referenceable("Manager", "SecurityClearance");
-
- hrDept.set("name", "hr");
- john.set("name", "John");
- john.set("department", hrDept);
- jane.set("name", "Jane");
- jane.set("department", hrDept);
-
- john.set("manager", jane);
-
- hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
-
- jane.set("subordinates", ImmutableList.<Referenceable>of(john));
-
- jane.getTrait("SecurityClearance").set("level", 1);
-
- ClassType deptType = ts.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
-
- String hrDeptStr = hrDept2.toString();
-
- Assert.assertEquals(hrDeptStr, "{\n" +
- "\tid : (type: Department, id: <unassigned>)\n" +
- "\tname : \thr\n" +
- "\temployees : \t[{\n" +
- "\tid : (type: Person, id: <unassigned>)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : (type: Manager, id: <unassigned>)\n" +
- "}, {\n" +
- "\tid : (type: Manager, id: <unassigned>)\n" +
- "\tsubordinates : \t[{\n" +
- "\tid : (type: Person, id: <unassigned>)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : (type: Manager, id: <unassigned>)\n" +
- "}]\n" +
- "\tname : \tJane\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : <null>\n" +
- "\n" +
- "\tSecurityClearance : \t{\n" +
- "\t\tlevel : \t\t1\n" +
- "\t}}]\n" +
- "}");
-
- String jsonStr = Serialization$.MODULE$.toJson(hrDept2);
- //System.out.println(jsonStr);
-
- hrDept2 = Serialization$.MODULE$.fromJson(jsonStr);
- Assert.assertEquals(hrDept2.toString(), hrDeptStr);
-
- }
-
- @Test
- public void testTrait() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
- HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef(
- "SecurityClearance2",
- ImmutableList.<String>of(),
- createRequiredAttrDef("level", DataTypes.INT_TYPE)
- );
-
- ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
- ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
-
-
- Struct s = new Struct("SecurityClearance2");
- s.set("level", 1);
- TraitType tType = ts.getDataType(TraitType.class, "SecurityClearance2");
- ITypedInstance t = tType.convert(s, Multiplicity.REQUIRED);
- String jsonStr = Serialization$.MODULE$.toJson(t);
- ITypedInstance t2 = Serialization$.MODULE$.traitFromJson(jsonStr);
- Assert.assertEquals(t.toString(), t2.toString());
- }
-}
\ No newline at end of file
[36/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
new file mode 100644
index 0000000..3a1f323
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import com.google.inject.Inject;
+import com.thinkaurelius.titan.core.TitanGraph;
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GraphTransactionInterceptor implements MethodInterceptor {
+ private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionInterceptor.class);
+ private TitanGraph titanGraph;
+
+ @Inject
+ GraphProvider<TitanGraph> graphProvider;
+
+ public Object invoke(MethodInvocation invocation) throws Throwable {
+ if (titanGraph == null) {
+ titanGraph = graphProvider.get();
+ }
+
+ try {
+ Object response = invocation.proceed();
+ titanGraph.commit();
+ LOG.debug("graph commit");
+ return response;
+ } catch (Throwable t){
+ titanGraph.rollback();
+ LOG.debug("graph rollback");
+ throw t;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
new file mode 100755
index 0000000..50de081
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import com.google.inject.matcher.Matchers;
+import com.google.inject.throwingproviders.ThrowingProviderBinder;
+import com.thinkaurelius.titan.core.TitanGraph;
+import org.aopalliance.intercept.MethodInterceptor;
+import org.apache.atlas.discovery.DiscoveryService;
+import org.apache.atlas.discovery.HiveLineageService;
+import org.apache.atlas.discovery.LineageService;
+import org.apache.atlas.discovery.SearchIndexer;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.repository.graph.TitanGraphProvider;
+import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
+import org.apache.atlas.repository.typestore.ITypeStore;
+import org.apache.atlas.services.DefaultMetadataService;
+import org.apache.atlas.services.MetadataService;
+
+/**
+ * Guice module for Repository module.
+ */
+public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
+ @Override
+ protected void configure() {
+ // special wiring for Titan Graph
+ ThrowingProviderBinder.create(binder())
+ .bind(GraphProvider.class, TitanGraph.class)
+ .to(TitanGraphProvider.class)
+ .asEagerSingleton();
+
+ // allow for dynamic binding of the metadata repo & graph service
+
+ // bind the MetadataRepositoryService interface to an implementation
+ bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton();
+
+ // bind the ITypeStore interface to an implementation
+ bind(ITypeStore.class).to(GraphBackedTypeStore.class).asEagerSingleton();
+
+ // bind the GraphService interface to an implementation
+ // bind(GraphService.class).to(graphServiceClass);
+
+ // bind the MetadataService interface to an implementation
+ bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
+
+ // bind the DiscoveryService interface to an implementation
+ bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
+
+ bind(SearchIndexer.class).to(GraphBackedSearchIndexer.class).asEagerSingleton();
+
+ bind(LineageService.class).to(HiveLineageService.class).asEagerSingleton();
+
+ MethodInterceptor interceptor = new GraphTransactionInterceptor();
+ requestInjection(interceptor);
+ bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
new file mode 100755
index 0000000..b5b4441
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.MetadataException;
+
+import java.security.PrivilegedActionException;
+
+public class DiscoveryException extends MetadataException {
+
+ /**
+ * Constructs a new exception with the specified detail message. The
+ * cause is not initialized, and may subsequently be initialized by
+ * a call to {@link #initCause}.
+ *
+ * @param message the detail message. The detail message is saved for
+ * later retrieval by the {@link #getMessage()} method.
+ */
+ public DiscoveryException(String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs a new exception with the specified detail message and
+ * cause. <p>Note that the detail message associated with
+ * {@code cause} is <i>not</i> automatically incorporated in
+ * this exception's detail message.
+ *
+ * @param message the detail message (which is saved for later retrieval
+ * by the {@link #getMessage()} method).
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A <tt>null</tt> value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public DiscoveryException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs a new exception with the specified cause and a detail
+ * message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ * typically contains the class and detail message of <tt>cause</tt>).
+ * This constructor is useful for exceptions that are little more than
+ * wrappers for other throwables (for example, {@link
+ * PrivilegedActionException}).
+ *
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A <tt>null</tt> value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public DiscoveryException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
new file mode 100755
index 0000000..e347c2c
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Metadata discovery service.
+ */
+public interface DiscoveryService {
+
+ /**
+ * Full text search
+ */
+ String searchByFullText(String query) throws DiscoveryException;
+
+ /**
+ * Search using query DSL.
+ *
+ * @param dslQuery query in DSL format.
+ * @return JSON representing the type and results.
+ */
+ String searchByDSL(String dslQuery) throws DiscoveryException;
+
+ /**
+ * Assumes the User is familiar with the persistence structure of the Repository.
+ * The given query is run uninterpreted against the underlying Graph Store.
+ * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
+ *
+ * @param gremlinQuery query in gremlin dsl format
+ * @return List of Maps
+ * @throws org.apache.atlas.discovery.DiscoveryException
+ */
+ List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
new file mode 100644
index 0000000..e97fcb8
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import org.apache.atlas.GraphTransaction;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.ParamChecker;
+import org.apache.atlas.PropertiesUtil;
+import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.query.Expressions;
+import org.apache.atlas.query.GremlinQueryResult;
+import org.apache.atlas.query.HiveLineageQuery;
+import org.apache.atlas.query.HiveWhereUsedQuery;
+import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Option;
+import scala.Some;
+import scala.collection.immutable.List;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+/**
+ * Hive implementation of Lineage service interface.
+ */
+@Singleton
+public class HiveLineageService implements LineageService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveLineageService.class);
+
+ private static final Option<List<String>> SELECT_ATTRIBUTES =
+ Some.<List<String>>apply(List.<String>fromArray(new String[]{"name"}));
+
+ private static final String HIVE_TABLE_TYPE_NAME;
+ private static final String HIVE_PROCESS_TYPE_NAME;
+ private static final String HIVE_PROCESS_INPUT_ATTRIBUTE_NAME;
+ private static final String HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME;
+
+ private static final String HIVE_TABLE_SCHEMA_QUERY;
+ private static final String HIVE_TABLE_EXISTS_QUERY;
+
+ static {
+ // todo - externalize this using type system - dog food
+ try {
+ PropertiesConfiguration conf = PropertiesUtil.getApplicationProperties();
+ HIVE_TABLE_TYPE_NAME =
+ conf.getString("atlas.lineage.hive.table.type.name", "DataSet");
+ HIVE_PROCESS_TYPE_NAME =
+ conf.getString("atlas.lineage.hive.process.type.name", "Process");
+ HIVE_PROCESS_INPUT_ATTRIBUTE_NAME =
+ conf.getString("atlas.lineage.hive.process.inputs.name", "inputs");
+ HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME =
+ conf.getString("atlas.lineage.hive.process.outputs.name", "outputs");
+
+ HIVE_TABLE_SCHEMA_QUERY = conf.getString(
+ "atlas.lineage.hive.table.schema.query",
+ "hive_table where name=\"%s\", columns");
+ HIVE_TABLE_EXISTS_QUERY = conf.getString(
+ "atlas.lineage.hive.table.exists.query",
+ "from hive_table where name=\"%s\"");
+ } catch (MetadataException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ private final TitanGraph titanGraph;
+ private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
+ private final GraphBackedDiscoveryService discoveryService;
+
+ @Inject
+ HiveLineageService(GraphProvider<TitanGraph> graphProvider,
+ MetadataRepository metadataRepository,
+ GraphBackedDiscoveryService discoveryService) throws DiscoveryException {
+ this.titanGraph = graphProvider.get();
+ this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
+ this.discoveryService = discoveryService;
+ }
+
+ /**
+ * Return the lineage outputs for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Lineage Outputs as JSON
+ */
+ @Override
+ @GraphTransaction
+ public String getOutputs(String tableName) throws MetadataException {
+ LOG.info("Fetching lineage outputs for tableName={}", tableName);
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ validateTableExists(tableName);
+
+ HiveWhereUsedQuery outputsQuery = new HiveWhereUsedQuery(
+ HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
+ HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
+ Option.empty(), SELECT_ATTRIBUTES, true,
+ graphPersistenceStrategy, titanGraph);
+
+ Expressions.Expression expression = outputsQuery.expr();
+ LOG.debug("Expression is [" + expression.toString() +"]");
+ try {
+ return discoveryService.evaluate(expression).toJson();
+ } catch (Exception e) { // unable to catch ExpressionException
+ throw new DiscoveryException("Invalid expression [" + expression.toString() + "]", e);
+ }
+ }
+
+ /**
+ * Return the lineage outputs graph for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Outputs Graph as JSON
+ */
+ @Override
+ @GraphTransaction
+ public String getOutputsGraph(String tableName) throws MetadataException {
+ LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ validateTableExists(tableName);
+
+ HiveWhereUsedQuery outputsQuery = new HiveWhereUsedQuery(
+ HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
+ HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
+ Option.empty(), SELECT_ATTRIBUTES, true,
+ graphPersistenceStrategy, titanGraph);
+ return outputsQuery.graph().toInstanceJson();
+ }
+
+ /**
+ * Return the lineage inputs for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Lineage Inputs as JSON
+ */
+ @Override
+ @GraphTransaction
+ public String getInputs(String tableName) throws MetadataException {
+ LOG.info("Fetching lineage inputs for tableName={}", tableName);
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ validateTableExists(tableName);
+
+ HiveLineageQuery inputsQuery = new HiveLineageQuery(
+ HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
+ HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
+ Option.empty(), SELECT_ATTRIBUTES, true,
+ graphPersistenceStrategy, titanGraph);
+
+ Expressions.Expression expression = inputsQuery.expr();
+ LOG.debug("Expression is [" + expression.toString() +"]");
+ try {
+ return discoveryService.evaluate(expression).toJson();
+ } catch (Exception e) { // unable to catch ExpressionException
+ throw new DiscoveryException("Invalid expression [" + expression.toString() + "]", e);
+ }
+ }
+
+ /**
+ * Return the lineage inputs graph for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Inputs Graph as JSON
+ */
+ @Override
+ @GraphTransaction
+ public String getInputsGraph(String tableName) throws MetadataException {
+ LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ validateTableExists(tableName);
+
+ HiveLineageQuery inputsQuery = new HiveLineageQuery(
+ HIVE_TABLE_TYPE_NAME, tableName, HIVE_PROCESS_TYPE_NAME,
+ HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME,
+ Option.empty(), SELECT_ATTRIBUTES, true,
+ graphPersistenceStrategy, titanGraph);
+ return inputsQuery.graph().toInstanceJson();
+ }
+
+ /**
+ * Return the schema for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Schema as JSON
+ */
+ @Override
+ @GraphTransaction
+ public String getSchema(String tableName) throws MetadataException {
+ LOG.info("Fetching schema for tableName={}", tableName);
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ validateTableExists(tableName);
+
+ final String schemaQuery = String.format(HIVE_TABLE_SCHEMA_QUERY, tableName);
+ return discoveryService.searchByDSL(schemaQuery);
+ }
+
+ /**
+ * Validate if indeed this is a table type and exists.
+ *
+ * @param tableName table name
+ */
+ private void validateTableExists(String tableName) throws MetadataException {
+ final String tableExistsQuery = String.format(HIVE_TABLE_EXISTS_QUERY, tableName);
+ GremlinQueryResult queryResult = discoveryService.evaluate(tableExistsQuery);
+ if (!(queryResult.rows().length() > 0)) {
+ throw new EntityNotFoundException(tableName + " does not exist");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/discovery/LineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/LineageService.java b/repository/src/main/java/org/apache/atlas/discovery/LineageService.java
new file mode 100644
index 0000000..12f1ca3
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/LineageService.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.MetadataException;
+
+/**
+ * Lineage service interface.
+ */
+public interface LineageService {
+
+ /**
+ * Return the lineage outputs for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Outputs as JSON
+ */
+ String getOutputs(String tableName) throws MetadataException;
+
+ /**
+ * Return the lineage outputs graph for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Outputs Graph as JSON
+ */
+ String getOutputsGraph(String tableName) throws MetadataException;
+
+ /**
+ * Return the lineage inputs for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Inputs as JSON
+ */
+ String getInputs(String tableName) throws MetadataException;
+
+ /**
+ * Return the lineage inputs graph for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Inputs Graph as JSON
+ */
+ String getInputsGraph(String tableName) throws MetadataException;
+
+ /**
+ * Return the schema for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Schema as JSON
+ */
+ String getSchema(String tableName) throws MetadataException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
new file mode 100755
index 0000000..9823fbd
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.listener.TypesChangeListener;
+
+
+/**
+ * Interface for indexing types.
+ */
+public interface SearchIndexer extends TypesChangeListener {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
new file mode 100755
index 0000000..e9c12cc
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery.graph;
+
+import com.thinkaurelius.titan.core.TitanVertex;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.query.Expressions;
+import org.apache.atlas.query.GraphPersistenceStrategies;
+import org.apache.atlas.query.GraphPersistenceStrategies$class;
+import org.apache.atlas.query.TypeUtils;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.collection.Traversable;
+
+import java.util.List;
+
+/**
+ * Default implementation of GraphPersistenceStrategy.
+ */
+public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategies {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(DefaultGraphPersistenceStrategy.class);
+
+ private final GraphBackedMetadataRepository metadataRepository;
+
+ public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) {
+ this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository;
+ }
+
+ @Override
+ public String typeAttributeName() {
+ return metadataRepository.getTypeAttributeName();
+ }
+
+ @Override
+ public String superTypeAttributeName() {
+ return metadataRepository.getSuperTypeAttributeName();
+ }
+
+ @Override
+ public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
+ return metadataRepository.getEdgeLabel(dataType, aInfo);
+ }
+
+ @Override
+ public String traitLabel(IDataType<?> dataType, String traitName) {
+ return metadataRepository.getTraitLabel(dataType, traitName);
+ }
+
+ @Override
+ public String fieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) {
+ try {
+ return metadataRepository.getFieldNameInVertex(dataType, aInfo);
+ } catch (MetadataException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public List<String> traitNames(TitanVertex vertex) {
+ return metadataRepository.getTraitNames(vertex);
+ }
+
+ @Override
+ public String fieldPrefixInSelect() {
+ return "it";
+ }
+
+ @Override
+ public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
+ return metadataRepository.getIdFromVertex(dataTypeName, vertex);
+ }
+
+ @Override
+ public <U> U constructInstance(IDataType<U> dataType, Object value) {
+ try {
+ switch (dataType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ return dataType.convert(value, Multiplicity.OPTIONAL);
+
+ case ARRAY:
+ // todo
+ break;
+
+ case MAP:
+ // todo
+ break;
+
+ case STRUCT:
+ TitanVertex structVertex = (TitanVertex) value;
+ StructType structType = (StructType) dataType;
+ ITypedStruct structInstance = structType.createInstance();
+
+ TypeSystem.IdType idType = TypeSystem.getInstance().getIdType();
+
+ if (dataType.getName().equals(idType.getName())) {
+ structInstance.set(idType.typeNameAttrName(),
+ structVertex.getProperty(typeAttributeName()));
+ structInstance.set(idType.idAttrName(),
+ structVertex.getProperty(idAttributeName()));
+
+ } else {
+ metadataRepository.getGraphToInstanceMapper().mapVertexToInstance(
+ structVertex, structInstance, structType.fieldMapping().fields);
+ }
+ return dataType.convert(structInstance, Multiplicity.OPTIONAL);
+
+ case TRAIT:
+ TitanVertex traitVertex = (TitanVertex) value;
+ TraitType traitType = (TraitType) dataType;
+ ITypedStruct traitInstance = traitType.createInstance();
+ // todo - this is not right, we should load the Instance associated with this
+ // trait. for now just loading the trait struct.
+ // metadataRepository.getGraphToInstanceMapper().mapVertexToTraitInstance(
+ // traitVertex, dataType.getName(), , traitType, traitInstance);
+ metadataRepository.getGraphToInstanceMapper().mapVertexToInstance(
+ traitVertex, traitInstance, traitType.fieldMapping().fields);
+ break;
+
+ case CLASS:
+ TitanVertex classVertex = (TitanVertex) value;
+ ITypedReferenceableInstance classInstance =
+ metadataRepository.getGraphToInstanceMapper().mapGraphToTypedInstance(
+ classVertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
+ classVertex);
+ return dataType.convert(classInstance, Multiplicity.OPTIONAL);
+
+ default:
+ throw new UnsupportedOperationException(
+ "Load for type " + dataType + "is not supported");
+ }
+ } catch (MetadataException e) {
+ LOG.error("error while constructing an instance", e);
+ }
+
+ return null;
+ }
+
+ @Override
+ public String edgeLabel(TypeUtils.FieldInfo fInfo) {
+ return fInfo.reverseDataType() == null
+ ? edgeLabel(fInfo.dataType(), fInfo.attrInfo())
+ : edgeLabel(fInfo.reverseDataType(), fInfo.attrInfo());
+ }
+
+ @Override
+ public String gremlinCompOp(Expressions.ComparisonExpression op) {
+ return GraphPersistenceStrategies$class.gremlinCompOp(this, op);
+ }
+
+ @Override
+ public String loopObjectExpression(IDataType<?> dataType) {
+ return GraphPersistenceStrategies$class.loopObjectExpression(this, dataType);
+ }
+
+ @Override
+ public String instanceToTraitEdgeDirection() { return "out"; }
+
+ @Override
+ public String traitToInstanceEdgeDirection() { return "in"; }
+
+ @Override
+ public String idAttributeName() { return metadataRepository.getIdAttributeName(); }
+
+ @Override
+ public scala.collection.Seq<String> typeTestExpression(String typeName, IntSequence intSeq) {
+ return GraphPersistenceStrategies$class.typeTestExpression(this, typeName, intSeq);
+ }
+
+ @Override
+ public boolean collectTypeInstancesIntoVar() {
+ return GraphPersistenceStrategies$class.collectTypeInstancesIntoVar(this);
+ }
+
+ @Override
+ public boolean addGraphVertexPrefix(scala.collection.Traversable<String> preStatements) {
+ return GraphPersistenceStrategies$class.addGraphVertexPrefix(this, preStatements);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
new file mode 100755
index 0000000..1314402
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery.graph;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanIndexQuery;
+import com.thinkaurelius.titan.core.TitanProperty;
+import com.thinkaurelius.titan.core.TitanVertex;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.GraphTransaction;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.discovery.DiscoveryException;
+import org.apache.atlas.discovery.DiscoveryService;
+import org.apache.atlas.query.Expressions;
+import org.apache.atlas.query.GremlinEvaluator;
+import org.apache.atlas.query.GremlinQuery;
+import org.apache.atlas.query.GremlinQueryResult;
+import org.apache.atlas.query.GremlinTranslator;
+import org.apache.atlas.query.QueryParser;
+import org.apache.atlas.query.QueryProcessor;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.util.Either;
+import scala.util.parsing.combinator.Parsers;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.script.Bindings;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Graph backed implementation of Search.
+ */
+@Singleton
+public class GraphBackedDiscoveryService implements DiscoveryService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class);
+
+ private final TitanGraph titanGraph;
+ private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
+
+ public final static String SCORE = "score";
+
+ @Inject
+ GraphBackedDiscoveryService(GraphProvider<TitanGraph> graphProvider,
+ MetadataRepository metadataRepository) throws DiscoveryException {
+ this.titanGraph = graphProvider.get();
+ this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
+ }
+
+ //Refer http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query
+ //http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query
+ // .html#query-string-syntax for query syntax
+ @Override
+ @GraphTransaction
+ public String searchByFullText(String query) throws DiscoveryException {
+ String graphQuery = String.format("v.%s:(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, query);
+ LOG.debug("Full text query: {}", graphQuery);
+ Iterator<TitanIndexQuery.Result<Vertex>> results =
+ titanGraph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices().iterator();
+ JSONArray response = new JSONArray();
+
+ while (results.hasNext()) {
+ TitanIndexQuery.Result<Vertex> result = results.next();
+ Vertex vertex = result.getElement();
+
+ JSONObject row = new JSONObject();
+ String guid = vertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ if (guid != null) { //Filter non-class entities
+ try {
+ row.put("guid", guid);
+ row.put(MetadataServiceClient.TYPENAME, vertex.<String>getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
+ row.put(SCORE, result.getScore());
+ } catch (JSONException e) {
+ LOG.error("Unable to create response", e);
+ throw new DiscoveryException("Unable to create response");
+ }
+
+ response.put(row);
+ }
+ }
+ return response.toString();
+ }
+
+ /**
+ * Search using query DSL.
+ *
+ * @param dslQuery query in DSL format.
+ * @return JSON representing the type and results.
+ */
+ @Override
+ @GraphTransaction
+ public String searchByDSL(String dslQuery) throws DiscoveryException {
+ LOG.info("Executing dsl query={}", dslQuery);
+ GremlinQueryResult queryResult = evaluate(dslQuery);
+ return queryResult.toJson();
+ }
+
+ public GremlinQueryResult evaluate(String dslQuery) throws DiscoveryException {
+ LOG.info("Executing dsl query={}", dslQuery);
+ try {
+ QueryParser queryParser = new QueryParser();
+ Either<Parsers.NoSuccess, Expressions.Expression> either = queryParser.apply(dslQuery);
+ if (either.isRight()) {
+ Expressions.Expression expression = either.right().get();
+ return evaluate(expression);
+ } else {
+ throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left());
+ }
+ } catch (Exception e) { // unable to catch ExpressionException
+ throw new DiscoveryException("Invalid expression : " + dslQuery, e);
+ }
+ }
+
+ public GremlinQueryResult evaluate(Expressions.Expression expression) {
+ Expressions.Expression validatedExpression = QueryProcessor.validate(expression);
+ GremlinQuery gremlinQuery =
+ new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate();
+ LOG.debug("Query = {}", validatedExpression);
+ LOG.debug("Expression Tree = {}", validatedExpression.treeString());
+ LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr());
+ return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, titanGraph).evaluate();
+ }
+
+ /**
+ * Assumes the User is familiar with the persistence structure of the Repository.
+ * The given query is run uninterpreted against the underlying Graph Store.
+ * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
+ *
+ * @param gremlinQuery query in gremlin dsl format
+ * @return List of Maps
+ * @throws org.apache.atlas.discovery.DiscoveryException
+ */
+ @Override
+ @GraphTransaction
+ public List<Map<String, String>> searchByGremlin(String gremlinQuery)
+ throws DiscoveryException {
+ LOG.info("Executing gremlin query={}", gremlinQuery);
+ ScriptEngineManager manager = new ScriptEngineManager();
+ ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
+ Bindings bindings = engine.createBindings();
+ bindings.put("g", titanGraph);
+
+ try {
+ Object o = engine.eval(gremlinQuery, bindings);
+ return extractResult(o);
+ } catch (ScriptException se) {
+ throw new DiscoveryException(se);
+ }
+ }
+
+ private List<Map<String, String>> extractResult(Object o) throws DiscoveryException {
+ if (!(o instanceof List)) {
+ throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
+ }
+
+ List l = (List) o;
+ List<Map<String, String>> result = new ArrayList<>();
+ for (Object r : l) {
+
+ Map<String, String> oRow = new HashMap<>();
+ if (r instanceof Map) {
+ @SuppressWarnings("unchecked")
+ Map<Object, Object> iRow = (Map) r;
+ for (Map.Entry e : iRow.entrySet()) {
+ Object k = e.getKey();
+ Object v = e.getValue();
+ oRow.put(k.toString(), v.toString());
+ }
+ } else if (r instanceof TitanVertex) {
+ Iterable<TitanProperty> ps = ((TitanVertex) r).getProperties();
+ for (TitanProperty tP : ps) {
+ String pName = tP.getPropertyKey().getName();
+ Object pValue = ((TitanVertex) r).getProperty(pName);
+ if (pValue != null) {
+ oRow.put(pName, pValue.toString());
+ }
+ }
+
+ } else if (r instanceof String) {
+ oRow.put("", r.toString());
+ } else {
+ throw new DiscoveryException(String.format("Cannot process result %s", o.toString()));
+ }
+
+ result.add(oRow);
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java b/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
new file mode 100755
index 0000000..1def3f8
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.listener;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+
+/**
+ * Entity (a Typed instance) change notification listener.
+ */
+public interface EntityChangeListener {
+
+ /**
+ * This is upon adding a new typed instance to the repository.
+ *
+ * @param typedInstance a typed instance
+ * @throws org.apache.atlas.MetadataException
+ */
+ void onEntityAdded(ITypedReferenceableInstance typedInstance) throws MetadataException;
+
+ /**
+ * This is upon adding a new trait to a typed instance.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitName trait name for the instance that needs to be added to entity
+ * @throws org.apache.atlas.MetadataException
+ */
+ void onTraitAdded(String guid, String traitName) throws MetadataException;
+
+ /**
+ * This is upon deleting a trait from a typed instance.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitName trait name for the instance that needs to be deleted from entity
+ * @throws org.apache.atlas.MetadataException
+ */
+ void onTraitDeleted(String guid, String traitName) throws MetadataException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/listener/TypesChangeListener.java b/repository/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
new file mode 100755
index 0000000..9cff874
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.listener;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.types.IDataType;
+
+/**
+ * Types change notification listener.
+ */
+public interface TypesChangeListener {
+
+ /**
+ * This is upon adding a new type to Store.
+ *
+ * @param typeName type name
+ * @param dataType data type
+ * @throws MetadataException
+ */
+ void onAdd(String typeName, IDataType dataType) throws MetadataException;
+
+ /**
+ * This is upon removing an existing type from the Store.
+ *
+ * @param typeName type name
+ * @throws MetadataException
+ */
+ // void onRemove(String typeName) throws MetadataException;
+
+ // This is upon updating an existing type to the store
+ // void onChange() throws MetadataException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java
new file mode 100755
index 0000000..9e3ec2d
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+public final class Constants {
+
+ /**
+ * Globally Unique identifier property key.
+ */
+
+ public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
+ public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid";
+ public static final String GUID_INDEX = "guid_index";
+
+ /**
+ * Entity type name property key.
+ */
+ public static final String ENTITY_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
+ public static final String ENTITY_TYPE_INDEX = "type_index";
+
+ /**
+ * Entity type's super types property key.
+ */
+ public static final String SUPER_TYPES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames";
+ public static final String SUPER_TYPES_INDEX = "super_types_index";
+
+ /**
+ * Full-text for the entity for enabling full-text search.
+ */
+ //weird issue in TitanDB if __ added to this property key. Not adding it for now
+ public static final String ENTITY_TEXT_PROPERTY_KEY = "entityText";
+
+ /**
+ * Properties for type store graph
+ */
+ public static final String TYPE_CATEGORY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.category";
+ public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type";
+ public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
+
+ /**
+ * Trait names property key and index name.
+ */
+ public static final String TRAIT_NAMES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "traitNames";
+ public static final String TRAIT_NAMES_INDEX = "trait_names_index";
+
+ public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
+ public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
+
+ /**
+ * search backing index name.
+ */
+ public static final String BACKING_INDEX = "search";
+
+ /**
+ * search backing index name for vertex keys.
+ */
+ public static final String VERTEX_INDEX = "vertex_index";
+
+ /**
+ * search backing index name for edge labels.
+ */
+ public static final String EDGE_INDEX = "edge_index";
+
+ public static final String FULLTEXT_INDEX = "fulltext_index";
+
+ private Constants() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/DiscoverInstances.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/DiscoverInstances.java b/repository/src/main/java/org/apache/atlas/repository/DiscoverInstances.java
new file mode 100755
index 0000000..f359e81
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/DiscoverInstances.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Graph walker implementation for discovering instances.
+ */
+public class DiscoverInstances implements ObjectGraphWalker.NodeProcessor {
+
+ public final Map<Id, Id> idToNewIdMap;
+ public final Map<Id, IReferenceableInstance> idToInstanceMap;
+ final IRepository repository;
+
+ public DiscoverInstances(IRepository repository) {
+ this.repository = repository;
+ idToNewIdMap = new HashMap<>();
+ idToInstanceMap = new HashMap<>();
+ }
+
+ @Override
+ public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
+
+ IReferenceableInstance ref = null;
+ Id id = null;
+
+ if (nd.attributeName == null) {
+ ref = (IReferenceableInstance) nd.instance;
+ id = ref.getId();
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ if (nd.value != null && (nd.value instanceof Id)) {
+ id = (Id) nd.value;
+ }
+ }
+
+ if (id != null) {
+ if (id.isUnassigned()) {
+ if (!idToNewIdMap.containsKey(id)) {
+ idToNewIdMap.put(id, repository.newId(id.className));
+ }
+ if (ref != null && idToInstanceMap.containsKey(ref)) {
+ // Oops
+ throw new RepositoryException(
+ String.format("Unexpected internal error: Id %s processed again", id));
+ }
+ if (ref != null) {
+ idToInstanceMap.put(id, ref);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java b/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java
new file mode 100644
index 0000000..7c85d1e
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+/**
+ * A simple wrapper for 404.
+ */
+public class EntityNotFoundException extends RepositoryException {
+ public EntityNotFoundException() {
+ }
+
+ public EntityNotFoundException(String message) {
+ super(message);
+ }
+
+ public EntityNotFoundException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public EntityNotFoundException(Throwable cause) {
+ super(cause);
+ }
+
+ public EntityNotFoundException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/IRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/IRepository.java b/repository/src/main/java/org/apache/atlas/repository/IRepository.java
new file mode 100755
index 0000000..20e37ee
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/IRepository.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.TraitType;
+
+import java.util.List;
+
+/**
+ * Metadata Repository interface.
+ */
+public interface IRepository {
+
+ ITypedReferenceableInstance create(IReferenceableInstance i) throws RepositoryException;
+
+ ITypedReferenceableInstance update(ITypedReferenceableInstance i) throws RepositoryException;
+
+ void delete(ITypedReferenceableInstance i) throws RepositoryException;
+
+ Id newId(String typeName);
+
+ ITypedReferenceableInstance get(Id id) throws RepositoryException;
+
+ void defineClass(ClassType type) throws RepositoryException;
+
+ void defineTrait(TraitType type) throws RepositoryException;
+
+ void defineTypes(List<HierarchicalType> types) throws RepositoryException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
new file mode 100755
index 0000000..0f2d1f7
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.IDataType;
+
+import java.util.List;
+
+/**
+ * An interface for persisting metadata into a blueprints enabled graph db.
+ */
+public interface MetadataRepository {
+
+ /**
+ * Returns the property key used to store entity type name.
+ *
+ * @return property key used to store entity type name.
+ */
+ String getTypeAttributeName();
+
+ /**
+ * Returns the property key used to store super type names.
+ *
+ * @return property key used to store super type names.
+ */
+ String getSuperTypeAttributeName();
+
+ /**
+ * Return the property key used to store a given traitName in the repository.
+ *
+ * @param dataType data type
+ * @param traitName trait name
+ * @return property key used to store a given traitName
+ */
+ String getTraitLabel(IDataType<?> dataType, String traitName);
+
+ /**
+ * Return the property key used to store a given attribute in the repository.
+ *
+ * @param dataType data type
+ * @param aInfo attribute info
+ * @return property key used to store a given attribute
+ */
+ String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws MetadataException;
+
+ /**
+ * Return the edge label for a given attribute in the repository.
+ *
+ * @param dataType data type
+ * @param aInfo attribute info
+ * @return edge label for a given attribute
+ */
+ String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
+
+ /**
+ * Creates an entity definition (instance) corresponding to a given type.
+ *
+ * @param entity entity (typed instance)
+ * @return a globally unique identifier
+ * @throws RepositoryException
+ */
+ String createEntity(IReferenceableInstance entity) throws RepositoryException;
+
+ /**
+ * Fetch the complete definition of an entity given its GUID.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return entity (typed instance) definition
+ * @throws RepositoryException
+ */
+ ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
+
+ /**
+ * Gets the list of entities for a given entity type.
+ *
+ * @param entityType name of a type which is unique
+ * @return a list of entity names for the given type
+ * @throws RepositoryException
+ */
+ List<String> getEntityList(String entityType) throws RepositoryException;
+
+ /**
+ * Deletes an entity definition (instance) corresponding to a given type.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return true if deleted else false
+ * @throws RepositoryException
+ */
+ // boolean deleteEntity(String guid) throws RepositoryException;
+
+ /**
+ * Updates an entity given its GUID with the attribute name and value.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param attributeName name of the attribute
+ * @param attributeValue value of the attribute
+ * @return an entity instance with updated state
+ * @throws RepositoryException
+ */
+ //ITypedReferenceableInstance updateEntity(String guid, String attributeName,
+ // String attributeValue) throws RepositoryException;
+
+
+ // Trait management functions
+ /**
+ * Gets the list of trait names for a given entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return a list of trait names for the given entity guid
+ * @throws RepositoryException
+ */
+ List<String> getTraitNames(String guid) throws MetadataException;
+
+ /**
+ * Adds a new trait to an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitInstance trait instance that needs to be added to entity
+ * @throws RepositoryException
+ */
+ void addTrait(String guid,
+ ITypedStruct traitInstance) throws RepositoryException;
+
+ /**
+ * Deletes a given trait from an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitNameToBeDeleted name of the trait
+ * @throws RepositoryException
+ */
+ void deleteTrait(String guid,
+ String traitNameToBeDeleted) throws RepositoryException;
+
+ /**
+ * Adds the property to the entity that corresponds to the GUID
+ * @param guid entity id
+ * @param property property name
+ * @param value property value
+ */
+ void updateEntity(String guid, String property, String value) throws RepositoryException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/RepositoryException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/RepositoryException.java b/repository/src/main/java/org/apache/atlas/repository/RepositoryException.java
new file mode 100755
index 0000000..8265a6f
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/RepositoryException.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+import org.apache.atlas.MetadataException;
+
+/**
+ * Base Exception class for Repository API.
+ */
+public class RepositoryException extends MetadataException {
+
+ public RepositoryException() {
+ }
+
+ public RepositoryException(String message) {
+ super(message);
+ }
+
+ public RepositoryException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public RepositoryException(Throwable cause) {
+ super(cause);
+ }
+
+ public RepositoryException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
[42/58] [abbrv] incubator-atlas git commit: Fix broken python tests
as a result of refactoring to atlas
Posted by ve...@apache.org.
Fix broken python tests as a result of refactoring to atlas
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4ad5c42d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4ad5c42d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4ad5c42d
Branch: refs/heads/master
Commit: 4ad5c42dd948ad7a81872855f5d8e3eb562ac054
Parents: 3071197
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Mon Jun 8 21:15:31 2015 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Mon Jun 8 21:15:31 2015 -0700
----------------------------------------------------------------------
.../atlas/discovery/graph/DefaultGraphPersistenceStrategy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4ad5c42d/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
index e9c12cc..c56b219 100755
--- a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
@@ -23,6 +23,7 @@ import org.apache.atlas.MetadataException;
import org.apache.atlas.query.Expressions;
import org.apache.atlas.query.GraphPersistenceStrategies;
import org.apache.atlas.query.GraphPersistenceStrategies$class;
+import org.apache.atlas.query.IntSequence;
import org.apache.atlas.query.TypeUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
@@ -38,7 +39,6 @@ import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.collection.Traversable;
import java.util.List;
[40/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
new file mode 100755
index 0000000..64fca66
--- /dev/null
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -0,0 +1,512 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.bridge;
+
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.MetadataServiceException;
+import org.apache.atlas.hive.model.HiveDataModelGenerator;
+import org.apache.atlas.hive.model.HiveDataTypes;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A Bridge Utility that imports metadata from the Hive Meta Store
+ * and registers then in DGI.
+ */
+public class HiveMetaStoreBridge {
+ private static final String DEFAULT_DGI_URL = "http://localhost:21000/";
+ public static final String HIVE_CLUSTER_NAME = "hive.cluster.name";
+ public static final String DEFAULT_CLUSTER_NAME = "primary";
+ private final String clusterName;
+
+ public static final String DGI_URL_PROPERTY = "hive.hook.dgi.url";
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
+
+ private final Hive hiveClient;
+ private final MetadataServiceClient metadataServiceClient;
+
+ /**
+ * Construct a HiveMetaStoreBridge.
+ * @param hiveConf hive conf
+ */
+ public HiveMetaStoreBridge(HiveConf hiveConf) throws Exception {
+ clusterName = hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+ hiveClient = Hive.get(hiveConf);
+ metadataServiceClient = new MetadataServiceClient(hiveConf.get(DGI_URL_PROPERTY, DEFAULT_DGI_URL));
+ }
+
+ public MetadataServiceClient getMetadataServiceClient() {
+ return metadataServiceClient;
+ }
+
+ public void importHiveMetadata() throws Exception {
+ LOG.info("Importing hive metadata");
+ importDatabases();
+ }
+
+ private void importDatabases() throws Exception {
+ List<String> databases = hiveClient.getAllDatabases();
+ for (String databaseName : databases) {
+ Referenceable dbReference = registerDatabase(databaseName);
+
+ importTables(databaseName, dbReference);
+ }
+ }
+
+ public Referenceable registerDatabase(String databaseName) throws Exception {
+ Referenceable dbRef = getDatabaseReference(databaseName, clusterName);
+ if (dbRef == null) {
+ LOG.info("Importing objects from databaseName : " + databaseName);
+ Database hiveDB = hiveClient.getDatabase(databaseName);
+
+ dbRef = new Referenceable(HiveDataTypes.HIVE_DB.getName());
+ dbRef.set(HiveDataModelGenerator.NAME, hiveDB.getName().toLowerCase());
+ dbRef.set(HiveDataModelGenerator.CLUSTER_NAME, clusterName);
+ dbRef.set("description", hiveDB.getDescription());
+ dbRef.set("locationUri", hiveDB.getLocationUri());
+ dbRef.set("parameters", hiveDB.getParameters());
+ dbRef.set("ownerName", hiveDB.getOwnerName());
+ if (hiveDB.getOwnerType() != null) {
+ dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
+ }
+
+ dbRef = createInstance(dbRef);
+ } else {
+ LOG.info("Database {} is already registered with id {}", databaseName, dbRef.getId().id);
+ }
+ return dbRef;
+ }
+
+ public Referenceable createInstance(Referenceable referenceable) throws Exception {
+ String typeName = referenceable.getTypeName();
+ LOG.debug("creating instance of type " + typeName);
+
+ String entityJSON = InstanceSerialization.toJson(referenceable, true);
+ LOG.debug("Submitting new entity {} = {}", referenceable.getTypeName(), entityJSON);
+ JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
+ String guid = jsonObject.getString(MetadataServiceClient.GUID);
+ LOG.debug("created instance for type " + typeName + ", guid: " + guid);
+
+ return new Referenceable(guid, referenceable.getTypeName(), null);
+ }
+
+ private void importTables(String databaseName, Referenceable databaseReferenceable) throws Exception {
+ List<String> hiveTables = hiveClient.getAllTables(databaseName);
+
+ for (String tableName : hiveTables) {
+ Referenceable tableReferenceable = registerTable(databaseReferenceable, databaseName, tableName);
+
+ // Import Partitions
+ Referenceable sdReferenceable = getSDForTable(databaseName, tableName);
+ registerPartitions(databaseName, tableName, tableReferenceable, sdReferenceable);
+
+ // Import Indexes
+ importIndexes(databaseName, tableName, databaseReferenceable, tableReferenceable);
+ }
+ }
+
+ /**
+ * Gets reference for the database
+ *
+ *
+ * @param databaseName database Name
+ * @param clusterName cluster name
+ * @return Reference for database if exists, else null
+ * @throws Exception
+ */
+ private Referenceable getDatabaseReference(String databaseName, String clusterName) throws Exception {
+ LOG.debug("Getting reference for database {}", databaseName);
+ String typeName = HiveDataTypes.HIVE_DB.getName();
+
+ String dslQuery = String.format("%s where %s = '%s' and %s = '%s'", typeName,
+ HiveDataModelGenerator.NAME, databaseName.toLowerCase(), HiveDataModelGenerator.CLUSTER_NAME,
+ clusterName);
+ return getEntityReferenceFromDSL(typeName, dslQuery);
+ }
+
+ public Referenceable getProcessReference(String queryStr) throws Exception {
+ LOG.debug("Getting reference for process with query {}", queryStr);
+ String typeName = HiveDataTypes.HIVE_PROCESS.getName();
+
+ //todo enable DSL
+// String dslQuery = String.format("%s where queryText = \"%s\"", typeName, queryStr);
+// return getEntityReferenceFromDSL(typeName, dslQuery);
+
+ String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.queryText', \"%s\").toList()",
+ typeName, typeName, StringEscapeUtils.escapeJava(queryStr));
+ return getEntityReferenceFromGremlin(typeName, gremlinQuery);
+ }
+
+ private Referenceable getEntityReferenceFromDSL(String typeName, String dslQuery) throws Exception {
+ MetadataServiceClient dgiClient = getMetadataServiceClient();
+ JSONArray results = dgiClient.searchByDSL(dslQuery);
+ if (results.length() == 0) {
+ return null;
+ } else {
+ String guid;
+ JSONObject row = results.getJSONObject(0);
+ if (row.has("$id$")) {
+ guid = row.getJSONObject("$id$").getString("id");
+ } else {
+ guid = row.getJSONObject("_col_0").getString("id");
+ }
+ return new Referenceable(guid, typeName, null);
+ }
+ }
+
+ public static String getTableName(String clusterName, String dbName, String tableName) {
+ return String.format("%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(), clusterName);
+ }
+
+ /**
+ * Gets reference for the table
+ *
+ * @param dbName database name
+ * @param tableName table name
+ * @return table reference if exists, else null
+ * @throws Exception
+ */
+ private Referenceable getTableReference(String dbName, String tableName) throws Exception {
+ LOG.debug("Getting reference for table {}.{}", dbName, tableName);
+
+ String typeName = HiveDataTypes.HIVE_TABLE.getName();
+ String entityName = getTableName(clusterName, dbName, tableName);
+ String dslQuery = String.format("%s as t where name = '%s'", typeName, entityName);
+ return getEntityReferenceFromDSL(typeName, dslQuery);
+ }
+
+ private Referenceable getEntityReferenceFromGremlin(String typeName, String gremlinQuery) throws MetadataServiceException,
+ JSONException {
+ MetadataServiceClient client = getMetadataServiceClient();
+ JSONObject response = client.searchByGremlin(gremlinQuery);
+ JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
+ if (results.length() == 0) {
+ return null;
+ }
+ String guid = results.getJSONObject(0).getString("__guid");
+ return new Referenceable(guid, typeName, null);
+ }
+
+ private Referenceable getPartitionReference(String dbName, String tableName, List<String> values) throws Exception {
+ String valuesStr = "['" + StringUtils.join(values, "', '") + "']";
+ LOG.debug("Getting reference for partition for {}.{} with values {}", dbName, tableName, valuesStr);
+ String typeName = HiveDataTypes.HIVE_PARTITION.getName();
+
+ //todo replace gremlin with DSL
+ // String dslQuery = String.format("%s as p where values = %s, tableName where name = '%s', "
+ // + "dbName where name = '%s' and clusterName = '%s' select p", typeName, valuesStr, tableName,
+ // dbName, clusterName);
+
+ String datasetType = MetadataServiceClient.DATA_SET_SUPER_TYPE;
+ String tableEntityName = getTableName(clusterName, dbName, tableName);
+
+ String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', %s).as('p')."
+ + "out('__%s.table').has('%s.name', '%s').back('p').toList()", typeName, typeName, valuesStr,
+ typeName, datasetType, tableEntityName);
+
+ return getEntityReferenceFromGremlin(typeName, gremlinQuery);
+ }
+
+ private Referenceable getSDForTable(String dbName, String tableName) throws Exception {
+ Referenceable tableRef = getTableReference(dbName, tableName);
+ if (tableRef == null) {
+ throw new IllegalArgumentException("Table " + dbName + "." + tableName + " doesn't exist");
+ }
+
+ MetadataServiceClient dgiClient = getMetadataServiceClient();
+ Referenceable tableInstance = dgiClient.getEntity(tableRef.getId().id);
+ Id sdId = (Id) tableInstance.get("sd");
+ return new Referenceable(sdId.id, sdId.getTypeName(), null);
+ }
+
+ public Referenceable registerTable(String dbName, String tableName) throws Exception {
+ Referenceable dbReferenceable = registerDatabase(dbName);
+ return registerTable(dbReferenceable, dbName, tableName);
+ }
+
+ public Referenceable registerTable(Referenceable dbReference, String dbName, String tableName) throws Exception {
+ LOG.info("Attempting to register table [" + tableName + "]");
+ Referenceable tableRef = getTableReference(dbName, tableName);
+ if (tableRef == null) {
+ LOG.info("Importing objects from " + dbName + "." + tableName);
+
+ Table hiveTable = hiveClient.getTable(dbName, tableName);
+
+ tableRef = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
+ tableRef.set(HiveDataModelGenerator.NAME,
+ getTableName(clusterName, hiveTable.getDbName(), hiveTable.getTableName()));
+ tableRef.set(HiveDataModelGenerator.TABLE_NAME, hiveTable.getTableName().toLowerCase());
+ tableRef.set("owner", hiveTable.getOwner());
+
+ tableRef.set("createTime", hiveTable.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME));
+ tableRef.set("lastAccessTime", hiveTable.getLastAccessTime());
+ tableRef.set("retention", hiveTable.getRetention());
+
+ tableRef.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT));
+
+ // add reference to the database
+ tableRef.set(HiveDataModelGenerator.DB, dbReference);
+
+ List<Referenceable> colList = getColumns(hiveTable.getCols());
+ tableRef.set("columns", colList);
+
+ // add reference to the StorageDescriptor
+ StorageDescriptor storageDesc = hiveTable.getSd();
+ Referenceable sdReferenceable = fillStorageDescStruct(storageDesc, colList);
+ tableRef.set("sd", sdReferenceable);
+
+ // add reference to the Partition Keys
+ List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys());
+ tableRef.set("partitionKeys", partKeys);
+
+ tableRef.set("parameters", hiveTable.getParameters());
+
+ if (hiveTable.getViewOriginalText() != null) {
+ tableRef.set("viewOriginalText", hiveTable.getViewOriginalText());
+ }
+
+ if (hiveTable.getViewExpandedText() != null) {
+ tableRef.set("viewExpandedText", hiveTable.getViewExpandedText());
+ }
+
+ tableRef.set("tableType", hiveTable.getTableType().name());
+ tableRef.set("temporary", hiveTable.isTemporary());
+
+
+ tableRef = createInstance(tableRef);
+ } else {
+ LOG.info("Table {}.{} is already registered with id {}", dbName, tableName, tableRef.getId().id);
+ }
+ return tableRef;
+ }
+
+ private void registerPartitions(String db, String tableName, Referenceable tableReferenceable,
+ Referenceable sdReferenceable) throws Exception {
+ Set<Partition> tableParts = hiveClient.getAllPartitionsOf(new Table(Table.getEmptyTable(db, tableName)));
+
+ if (tableParts.size() > 0) {
+ for (Partition hivePart : tableParts) {
+ registerPartition(hivePart, tableReferenceable, sdReferenceable);
+ }
+ }
+ }
+
+ public Referenceable registerPartition(Partition partition) throws Exception {
+ String dbName = partition.getTable().getDbName();
+ String tableName = partition.getTable().getTableName();
+ Referenceable tableRef = registerTable(dbName, tableName);
+ Referenceable sdRef = getSDForTable(dbName, tableName);
+ return registerPartition(partition, tableRef, sdRef);
+ }
+
+ private Referenceable registerPartition(Partition hivePart, Referenceable tableReferenceable,
+ Referenceable sdReferenceable) throws Exception {
+ LOG.info("Registering partition for {} with values {}", tableReferenceable,
+ StringUtils.join(hivePart.getValues(), ","));
+ String dbName = hivePart.getTable().getDbName();
+ String tableName = hivePart.getTable().getTableName();
+
+ Referenceable partRef = getPartitionReference(dbName, tableName, hivePart.getValues());
+ if (partRef == null) {
+ partRef = new Referenceable(HiveDataTypes.HIVE_PARTITION.getName());
+ partRef.set("values", hivePart.getValues());
+
+ partRef.set(HiveDataModelGenerator.TABLE, tableReferenceable);
+
+ //todo fix
+ partRef.set("createTime", hivePart.getLastAccessTime());
+ partRef.set("lastAccessTime", hivePart.getLastAccessTime());
+
+ // sdStruct = fillStorageDescStruct(hivePart.getSd());
+ // Instead of creating copies of the sdstruct for partitions we are reusing existing
+ // ones will fix to identify partitions with differing schema.
+ partRef.set("sd", sdReferenceable);
+
+ partRef.set("parameters", hivePart.getParameters());
+ partRef = createInstance(partRef);
+ } else {
+ LOG.info("Partition {}.{} with values {} is already registered with id {}", dbName, tableName,
+ StringUtils.join(hivePart.getValues(), ","), partRef.getId().id);
+ }
+ return partRef;
+ }
+
+ private void importIndexes(String db, String table,
+ Referenceable dbReferenceable,
+ Referenceable tableReferenceable) throws Exception {
+ List<Index> indexes = hiveClient.getIndexes(db, table, Short.MAX_VALUE);
+ if (indexes.size() > 0) {
+ for (Index index : indexes) {
+ importIndex(index, dbReferenceable, tableReferenceable);
+ }
+ }
+ }
+
+ //todo should be idempotent
+ private void importIndex(Index index,
+ Referenceable dbReferenceable,
+ Referenceable tableReferenceable) throws Exception {
+ LOG.info("Importing index {} for {}.{}", index.getIndexName(), dbReferenceable, tableReferenceable);
+ Referenceable indexRef = new Referenceable(HiveDataTypes.HIVE_INDEX.getName());
+
+ indexRef.set(HiveDataModelGenerator.NAME, index.getIndexName());
+ indexRef.set("indexHandlerClass", index.getIndexHandlerClass());
+
+ indexRef.set(HiveDataModelGenerator.DB, dbReferenceable);
+
+ indexRef.set("createTime", index.getCreateTime());
+ indexRef.set("lastAccessTime", index.getLastAccessTime());
+ indexRef.set("origTable", index.getOrigTableName());
+ indexRef.set("indexTable", index.getIndexTableName());
+
+ Referenceable sdReferenceable = fillStorageDescStruct(index.getSd(), null);
+ indexRef.set("sd", sdReferenceable);
+
+ indexRef.set("parameters", index.getParameters());
+
+ tableReferenceable.set("deferredRebuild", index.isDeferredRebuild());
+
+ createInstance(indexRef);
+ }
+
+ private Referenceable fillStorageDescStruct(StorageDescriptor storageDesc, List<Referenceable> colList) throws Exception {
+ LOG.debug("Filling storage descriptor information for " + storageDesc);
+
+ Referenceable sdReferenceable = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
+
+ SerDeInfo serdeInfo = storageDesc.getSerdeInfo();
+ LOG.debug("serdeInfo = " + serdeInfo);
+ // SkewedInfo skewedInfo = storageDesc.getSkewedInfo();
+
+ String serdeInfoName = HiveDataTypes.HIVE_SERDE.getName();
+ Struct serdeInfoStruct = new Struct(serdeInfoName);
+
+ serdeInfoStruct.set(HiveDataModelGenerator.NAME, serdeInfo.getName());
+ serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
+ serdeInfoStruct.set("parameters", serdeInfo.getParameters());
+
+ sdReferenceable.set("serdeInfo", serdeInfoStruct);
+ sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
+ sdReferenceable.set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
+
+ //Use the passed column list if not null, ex: use same references for table and SD
+ List<FieldSchema> columns = storageDesc.getCols();
+ if (columns != null && !columns.isEmpty()) {
+ if (colList != null) {
+ sdReferenceable.set("cols", colList);
+ } else {
+ sdReferenceable.set("cols", getColumns(columns));
+ }
+ }
+
+ List<Struct> sortColsStruct = new ArrayList<>();
+ for (Order sortcol : storageDesc.getSortCols()) {
+ String hiveOrderName = HiveDataTypes.HIVE_ORDER.getName();
+ Struct colStruct = new Struct(hiveOrderName);
+ colStruct.set("col", sortcol.getCol());
+ colStruct.set("order", sortcol.getOrder());
+
+ sortColsStruct.add(colStruct);
+ }
+ if (sortColsStruct.size() > 0) {
+ sdReferenceable.set("sortCols", sortColsStruct);
+ }
+
+ sdReferenceable.set("location", storageDesc.getLocation());
+ sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
+ sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
+ sdReferenceable.set("compressed", storageDesc.isCompressed());
+
+ if (storageDesc.getBucketCols().size() > 0) {
+ sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
+ }
+
+ sdReferenceable.set("parameters", storageDesc.getParameters());
+ sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
+
+ return createInstance(sdReferenceable);
+ }
+
+ private List<Referenceable> getColumns(List<FieldSchema> schemaList) throws Exception
+ {
+ List<Referenceable> colList = new ArrayList<>();
+ for (FieldSchema fs : schemaList) {
+ LOG.debug("Processing field " + fs);
+ Referenceable colReferenceable = new Referenceable(HiveDataTypes.HIVE_COLUMN.getName());
+ colReferenceable.set(HiveDataModelGenerator.NAME, fs.getName());
+ colReferenceable.set("type", fs.getType());
+ colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment());
+
+ colList.add(createInstance(colReferenceable));
+ }
+ return colList;
+ }
+
+ public synchronized void registerHiveDataModel() throws Exception {
+ HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
+ MetadataServiceClient dgiClient = getMetadataServiceClient();
+
+ //Register hive data model if its not already registered
+ if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null ) {
+ LOG.info("Registering Hive data model");
+ dgiClient.createType(dataModelGenerator.getModelAsJson());
+ } else {
+ LOG.info("Hive data model is already registered!");
+ }
+ }
+
+ public static void main(String[] argv) throws Exception {
+ HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf());
+ hiveMetaStoreBridge.registerHiveDataModel();
+ hiveMetaStoreBridge.importHiveMetadata();
+ }
+
+ public void updateTable(Referenceable tableReferenceable, Table newTable) throws MetadataServiceException {
+ MetadataServiceClient client = getMetadataServiceClient();
+ client.updateEntity(tableReferenceable.getId()._getId(), HiveDataModelGenerator.TABLE_NAME,
+ newTable.getTableName().toLowerCase());
+ client.updateEntity(tableReferenceable.getId()._getId(), HiveDataModelGenerator.NAME,
+ getTableName(clusterName, newTable.getDbName(), newTable.getTableName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
new file mode 100755
index 0000000..a70bfea
--- /dev/null
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -0,0 +1,339 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.hook;
+
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
+import org.apache.atlas.hive.model.HiveDataTypes;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.ExplainTask;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.hooks.Entity;
+import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * DgiHook sends lineage information to the DgiSever.
+ */
+public class HiveHook implements ExecuteWithHookContext {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveHook.class);
+
+ // wait time determines how long we wait before we exit the jvm on
+ // shutdown. Pending requests after that will not be sent.
+ private static final int WAIT_TIME = 3;
+ private static ExecutorService executor;
+
+ private static final String MIN_THREADS = "hive.hook.dgi.minThreads";
+ private static final String MAX_THREADS = "hive.hook.dgi.maxThreads";
+ private static final String KEEP_ALIVE_TIME = "hive.hook.dgi.keepAliveTime";
+
+ private static final int minThreadsDefault = 5;
+ private static final int maxThreadsDefault = 5;
+ private static final long keepAliveTimeDefault = 10;
+ private static boolean typesRegistered = false;
+
+ static {
+ // anything shared should be initialized here and destroyed in the
+ // shutdown hook The hook contract is weird in that it creates a
+ // boatload of hooks.
+
+ // initialize the async facility to process hook calls. We don't
+ // want to do this inline since it adds plenty of overhead for the
+ // query.
+ HiveConf hiveConf = new HiveConf();
+ int minThreads = hiveConf.getInt(MIN_THREADS, minThreadsDefault);
+ int maxThreads = hiveConf.getInt(MAX_THREADS, maxThreadsDefault);
+ long keepAliveTime = hiveConf.getLong(KEEP_ALIVE_TIME, keepAliveTimeDefault);
+
+ executor = new ThreadPoolExecutor(minThreads, maxThreads, keepAliveTime, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(),
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("DGI Logger %d").build());
+
+ try {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ try {
+ executor.shutdown();
+ executor.awaitTermination(WAIT_TIME, TimeUnit.SECONDS);
+ executor = null;
+ } catch (InterruptedException ie) {
+ LOG.info("Interrupt received in shutdown.");
+ }
+ // shutdown client
+ }
+ });
+ } catch (IllegalStateException is) {
+ LOG.info("Attempting to send msg while shutdown in progress.");
+ }
+
+ LOG.info("Created DGI Hook");
+ }
+
+ class HiveEvent {
+ public HiveConf conf;
+
+ public Set<ReadEntity> inputs;
+ public Set<WriteEntity> outputs;
+
+ public String user;
+ public HiveOperation operation;
+ public QueryPlan queryPlan;
+ public HookContext.HookType hookType;
+ public JSONObject jsonPlan;
+ }
+
+ @Override
+ public void run(final HookContext hookContext) throws Exception {
+ if (executor == null) {
+ LOG.info("No executor running. Bail.");
+ return;
+ }
+
+ // clone to avoid concurrent access
+ final HiveEvent event = new HiveEvent();
+ final HiveConf conf = new HiveConf(hookContext.getConf());
+ boolean debug = conf.get("hive.hook.dgi.synchronous", "false").equals("true");
+
+ event.conf = conf;
+ event.inputs = hookContext.getInputs();
+ event.outputs = hookContext.getOutputs();
+
+ event.user = hookContext.getUserName() == null ? hookContext.getUgi().getUserName() : hookContext.getUserName();
+ event.operation = HiveOperation.valueOf(hookContext.getOperationName());
+ event.queryPlan = hookContext.getQueryPlan();
+ event.hookType = hookContext.getHookType();
+
+ event.jsonPlan = getQueryPlan(event);
+
+ if (debug) {
+ fireAndForget(event);
+ } else {
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ fireAndForget(event);
+ } catch (Throwable e) {
+ LOG.info("DGI hook failed", e);
+ }
+ }
+ });
+ }
+ }
+
+ private void fireAndForget(HiveEvent event) throws Exception {
+ assert event.hookType == HookContext.HookType.POST_EXEC_HOOK : "Non-POST_EXEC_HOOK not supported!";
+
+ LOG.info("Entered DGI hook for hook type {} operation {}", event.hookType, event.operation);
+ HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(event.conf);
+
+ if (!typesRegistered) {
+ dgiBridge.registerHiveDataModel();
+ typesRegistered = true;
+ }
+
+ switch (event.operation) {
+ case CREATEDATABASE:
+ handleCreateDB(dgiBridge, event);
+ break;
+
+ case CREATETABLE:
+ handleCreateTable(dgiBridge, event);
+ break;
+
+ case CREATETABLE_AS_SELECT:
+ case CREATEVIEW:
+ case LOAD:
+ case EXPORT:
+ case IMPORT:
+ case QUERY:
+ registerProcess(dgiBridge, event);
+ break;
+
+ case ALTERTABLE_RENAME:
+ case ALTERVIEW_RENAME:
+ renameTable(dgiBridge, event);
+ break;
+
+ case ALTERVIEW_AS:
+ //update inputs/outputs?
+ break;
+
+ case ALTERTABLE_ADDCOLS:
+ case ALTERTABLE_REPLACECOLS:
+ case ALTERTABLE_RENAMECOL:
+ break;
+
+ default:
+ }
+ }
+
+ private void renameTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
+ //crappy, no easy of getting new name
+ assert event.inputs != null && event.inputs.size() == 1;
+ assert event.outputs != null && event.outputs.size() > 0;
+
+ Table oldTable = event.inputs.iterator().next().getTable();
+ Table newTable = null;
+ for (WriteEntity writeEntity : event.outputs) {
+ if (writeEntity.getType() == Entity.Type.TABLE) {
+ Table table = writeEntity.getTable();
+ if (table.getDbName().equals(oldTable.getDbName()) && !table.getTableName()
+ .equals(oldTable.getTableName())) {
+ newTable = table;
+ break;
+ }
+ }
+ }
+ if (newTable == null) {
+ LOG.warn("Failed to deduct new name for " + event.queryPlan.getQueryStr());
+ return;
+ }
+
+ Referenceable dbReferenceable = dgiBridge.registerDatabase(oldTable.getDbName());
+ Referenceable tableReferenceable =
+ dgiBridge.registerTable(dbReferenceable, oldTable.getDbName(), oldTable.getTableName());
+ LOG.info("Updating entity name {}.{} to {}", oldTable.getDbName(), oldTable.getTableName(),
+ newTable.getTableName());
+ dgiBridge.updateTable(tableReferenceable, newTable);
+ }
+
+ private void handleCreateTable(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
+ for (WriteEntity entity : event.outputs) {
+ if (entity.getType() == Entity.Type.TABLE) {
+
+ Table table = entity.getTable();
+ Referenceable dbReferenceable = dgiBridge.registerDatabase(table.getDbName());
+ dgiBridge.registerTable(dbReferenceable, table.getDbName(), table.getTableName());
+ }
+ }
+ }
+
+ private void handleCreateDB(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
+ for (WriteEntity entity : event.outputs) {
+ if (entity.getType() == Entity.Type.DATABASE) {
+ dgiBridge.registerDatabase(entity.getDatabase().getName());
+ }
+ }
+ }
+
+ private String normalize(String str) {
+ if (StringUtils.isEmpty(str)) {
+ return null;
+ }
+ return str.toLowerCase().trim();
+ }
+
+ private void registerProcess(HiveMetaStoreBridge dgiBridge, HiveEvent event) throws Exception {
+ Set<ReadEntity> inputs = event.inputs;
+ Set<WriteEntity> outputs = event.outputs;
+
+ //Even explain CTAS has operation name as CREATETABLE_AS_SELECT
+ if (inputs.isEmpty() && outputs.isEmpty()) {
+ LOG.info("Explain statement. Skipping...");
+ }
+
+ if (event.queryPlan == null) {
+ LOG.info("Query plan is missing. Skipping...");
+ }
+
+ String queryId = event.queryPlan.getQueryId();
+ String queryStr = normalize(event.queryPlan.getQueryStr());
+ long queryStartTime = event.queryPlan.getQueryStartTime();
+
+ LOG.debug("Registering CTAS query: {}", queryStr);
+ Referenceable processReferenceable = dgiBridge.getProcessReference(queryStr);
+ if (processReferenceable == null) {
+ processReferenceable = new Referenceable(HiveDataTypes.HIVE_PROCESS.getName());
+ processReferenceable.set("name", event.operation.getOperationName());
+ processReferenceable.set("startTime", queryStartTime);
+ processReferenceable.set("userName", event.user);
+
+ List<Referenceable> source = new ArrayList<>();
+ for (ReadEntity readEntity : inputs) {
+ if (readEntity.getType() == Entity.Type.TABLE) {
+ Table table = readEntity.getTable();
+ String dbName = table.getDbName();
+ source.add(dgiBridge.registerTable(dbName, table.getTableName()));
+ }
+ if (readEntity.getType() == Entity.Type.PARTITION) {
+ dgiBridge.registerPartition(readEntity.getPartition());
+ }
+ }
+ processReferenceable.set("inputs", source);
+
+ List<Referenceable> target = new ArrayList<>();
+ for (WriteEntity writeEntity : outputs) {
+ if (writeEntity.getType() == Entity.Type.TABLE || writeEntity.getType() == Entity.Type.PARTITION) {
+ Table table = writeEntity.getTable();
+ String dbName = table.getDbName();
+ target.add(dgiBridge.registerTable(dbName, table.getTableName()));
+ }
+ if (writeEntity.getType() == Entity.Type.PARTITION) {
+ dgiBridge.registerPartition(writeEntity.getPartition());
+ }
+ }
+ processReferenceable.set("outputs", target);
+ processReferenceable.set("queryText", queryStr);
+ processReferenceable.set("queryId", queryId);
+ processReferenceable.set("queryPlan", event.jsonPlan.toString());
+ processReferenceable.set("endTime", System.currentTimeMillis());
+
+ //TODO set
+ processReferenceable.set("queryGraph", "queryGraph");
+ dgiBridge.createInstance(processReferenceable);
+ } else {
+ LOG.debug("Query {} is already registered", queryStr);
+ }
+ }
+
+
+ private JSONObject getQueryPlan(HiveEvent event) throws Exception {
+ try {
+ ExplainTask explain = new ExplainTask();
+ explain.initialize(event.conf, event.queryPlan, null);
+ List<Task<?>> rootTasks = event.queryPlan.getRootTasks();
+ return explain.getJSONPlan(null, null, rootTasks, event.queryPlan.getFetchTask(), true, false, false);
+ } catch(Exception e) {
+ LOG.warn("Failed to get queryplan", e);
+ return new JSONObject();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
new file mode 100755
index 0000000..52e07fb
--- /dev/null
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.model;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Utility that generates hive data model for both metastore entities and DDL/DML queries.
+ */
+public class HiveDataModelGenerator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class);
+
+ private static final DataTypes.MapType STRING_MAP_TYPE =
+ new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
+
+ private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
+ private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
+ private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
+
+ public static final String COMMENT = "comment";
+
+ public static final String STORAGE_NUM_BUCKETS = "numBuckets";
+ public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
+
+ public static final String NAME = "name";
+ public static final String TABLE_NAME = "tableName";
+ public static final String CLUSTER_NAME = "clusterName";
+ public static final String TABLE = "table";
+ public static final String DB = "db";
+
+ public HiveDataModelGenerator() {
+ classTypeDefinitions = new HashMap<>();
+ enumTypeDefinitionMap = new HashMap<>();
+ structTypeDefinitionMap = new HashMap<>();
+ }
+
+ public void createDataModel() throws MetadataException {
+ LOG.info("Generating the Hive Data Model....");
+
+ // enums
+ createHiveObjectTypeEnum();
+ createHivePrincipalTypeEnum();
+ createResourceTypeEnum();
+
+ // structs
+ createSerDeStruct();
+ //createSkewedInfoStruct();
+ createOrderStruct();
+ createResourceUriStruct();
+ createStorageDescClass();
+
+ // classes
+ createDBClass();
+ createTypeClass();
+ createColumnClass();
+ createPartitionClass();
+ createTableClass();
+ createIndexClass();
+ createRoleClass();
+
+ // DDL/DML Process
+ createProcessClass();
+ }
+
+ public TypesDef getTypesDef() {
+ return TypeUtils.getTypesDef(
+ getEnumTypeDefinitions(),
+ getStructTypeDefinitions(),
+ getTraitTypeDefinitions(),
+ getClassTypeDefinitions()
+ );
+ }
+
+ public String getDataModelAsJSON() {
+ return TypesSerialization.toJson(getTypesDef());
+ }
+
+ public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
+ return ImmutableList.copyOf(enumTypeDefinitionMap.values());
+ }
+
+ public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
+ return ImmutableList.copyOf(structTypeDefinitionMap.values());
+ }
+
+ public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
+ return ImmutableList.copyOf(classTypeDefinitions.values());
+ }
+
+ public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
+ return ImmutableList.of();
+ }
+
+ private void createHiveObjectTypeEnum() throws MetadataException {
+ EnumValue values[] = {
+ new EnumValue("GLOBAL", 1),
+ new EnumValue("DATABASE", 2),
+ new EnumValue("TABLE", 3),
+ new EnumValue("PARTITION", 4),
+ new EnumValue("COLUMN", 5),
+ };
+
+ EnumTypeDefinition definition = new EnumTypeDefinition(
+ HiveDataTypes.HIVE_OBJECT_TYPE.getName(), values);
+ enumTypeDefinitionMap.put(HiveDataTypes.HIVE_OBJECT_TYPE.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_OBJECT_TYPE.getName());
+ }
+
+ private void createHivePrincipalTypeEnum() throws MetadataException {
+ EnumValue values[] = {
+ new EnumValue("USER", 1),
+ new EnumValue("ROLE", 2),
+ new EnumValue("GROUP", 3),
+ };
+
+ EnumTypeDefinition definition = new EnumTypeDefinition(
+ HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values);
+
+ enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
+ }
+
+ private void createResourceTypeEnum() throws MetadataException {
+ EnumValue values[] = {
+ new EnumValue("JAR", 1),
+ new EnumValue("FILE", 2),
+ new EnumValue("ARCHIVE", 3),
+ };
+ EnumTypeDefinition definition = new EnumTypeDefinition(
+ HiveDataTypes.HIVE_RESOURCE_TYPE.getName(), values);
+ enumTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCE_TYPE.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCE_TYPE.getName());
+ }
+
+ private void createSerDeStruct() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+ StructTypeDefinition definition = new StructTypeDefinition(
+ HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions);
+ structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
+ }
+
+ private void createOrderStruct() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("order", DataTypes.INT_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ };
+
+ StructTypeDefinition definition = new StructTypeDefinition(
+ HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions);
+ structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName());
+ }
+
+ private void createStorageDescClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("cols",
+ String.format("array<%s>", HiveDataTypes.HIVE_COLUMN.getName()),
+ Multiplicity.COLLECTION, false, null),
+ new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition(STORAGE_NUM_BUCKETS, DataTypes.INT_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("bucketCols",
+ String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("sortCols",
+ String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ //new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
+ // Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition(STORAGE_IS_STORED_AS_SUB_DIRS, DataTypes.BOOLEAN_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+
+ HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
+ ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null, attributeDefinitions);
+ classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
+ }
+
+ /** Revisit later after nested array types are handled by the typesystem **/
+
+ private void createResourceUriStruct() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("resourceType", HiveDataTypes.HIVE_RESOURCE_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("uri", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ };
+ StructTypeDefinition definition = new StructTypeDefinition(
+ HiveDataTypes.HIVE_RESOURCEURI.getName(), attributeDefinitions);
+ structTypeDefinitionMap.put(HiveDataTypes.HIVE_RESOURCEURI.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_RESOURCEURI.getName());
+ }
+
+ private void createDBClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition(CLUSTER_NAME, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("description", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("locationUri", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+
+ HierarchicalTypeDefinition<ClassType> definition =
+ new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(),
+ null, attributeDefinitions);
+ classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
+ }
+
+ private void createTypeClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("type1", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("type2", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("fields", String.format("array<%s>",
+ HiveDataTypes.HIVE_COLUMN.getName()), Multiplicity.OPTIONAL, false, null),
+ };
+ HierarchicalTypeDefinition<ClassType> definition =
+ new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TYPE.getName(),
+ null, attributeDefinitions);
+
+ classTypeDefinitions.put(HiveDataTypes.HIVE_TYPE.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_TYPE.getName());
+ }
+
+ private void createColumnClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+ HierarchicalTypeDefinition<ClassType> definition =
+ new HierarchicalTypeDefinition<>(
+ ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(),
+ null, attributeDefinitions);
+ classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
+ }
+
+ private void createPartitionClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("columns",
+ DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
+ Multiplicity.OPTIONAL, true, null),
+ new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+ HierarchicalTypeDefinition<ClassType> definition =
+ new HierarchicalTypeDefinition<>(ClassType.class,
+ HiveDataTypes.HIVE_PARTITION.getName(), null, attributeDefinitions);
+ classTypeDefinitions.put(HiveDataTypes.HIVE_PARTITION.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_PARTITION.getName());
+ }
+
+ private void createTableClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition(TABLE_NAME, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("partitionKeys",
+ DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("columns",
+ DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
+ Multiplicity.OPTIONAL, true, null),
+ new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("tableType", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+ HierarchicalTypeDefinition<ClassType> definition =
+ new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(),
+ ImmutableList.of("DataSet"), attributeDefinitions);
+ classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
+ }
+
+ private void createIndexClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition(NAME, DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("indexHandlerClass", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("origTable", HiveDataTypes.HIVE_TABLE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("indexTable", HiveDataTypes.HIVE_TABLE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("sd", HiveDataTypes.HIVE_STORAGEDESC.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition("deferredRebuild", DataTypes.BOOLEAN_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+
+ HierarchicalTypeDefinition<ClassType> definition =
+ new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_INDEX.getName(),
+ ImmutableList.of(MetadataServiceClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
+ classTypeDefinitions.put(HiveDataTypes.HIVE_INDEX.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_INDEX.getName());
+ }
+
+ private void createRoleClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("roleName", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("ownerName", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ };
+ HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
+ ClassType.class, HiveDataTypes.HIVE_ROLE.getName(), null, attributeDefinitions);
+
+ classTypeDefinitions.put(HiveDataTypes.HIVE_ROLE.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_ROLE.getName());
+ }
+
+ private void createProcessClass() throws MetadataException {
+ AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
+ new AttributeDefinition("startTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("endTime", DataTypes.LONG_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(),
+ Multiplicity.OPTIONAL, false, null),
+ };
+
+ HierarchicalTypeDefinition<ClassType> definition = new HierarchicalTypeDefinition<>(
+ ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(),
+ ImmutableList.of(MetadataServiceClient.PROCESS_SUPER_TYPE), attributeDefinitions);
+ classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
+ LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
+ }
+
+ public String getModelAsJson() throws MetadataException {
+ createDataModel();
+ return getDataModelAsJSON();
+ }
+
+ public static void main(String[] args) throws Exception {
+ HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
+ System.out.println("hiveDataModelAsJSON = " + hiveDataModelGenerator.getModelAsJson());
+
+ TypesDef typesDef = hiveDataModelGenerator.getTypesDef();
+ for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
+ System.out.println(String.format("%s(%s) - %s", enumType.name, EnumType.class.getSimpleName(),
+ Arrays.toString(enumType.enumValues)));
+ }
+ for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
+ System.out.println(String.format("%s(%s) - %s", structType.typeName, StructType.class.getSimpleName(),
+ Arrays.toString(structType.attributeDefinitions)));
+ }
+ for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
+ System.out.println(String.format("%s(%s) - %s", classType.typeName, ClassType.class.getSimpleName(),
+ Arrays.toString(classType.attributeDefinitions)));
+ }
+ for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
+ System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
+ Arrays.toString(traitType.attributeDefinitions)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataTypes.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataTypes.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataTypes.java
new file mode 100755
index 0000000..f376e6e
--- /dev/null
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataTypes.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.model;
+
+/**
+ * Hive Data Types for model and bridge.
+ */
+public enum HiveDataTypes {
+
+ // Enums
+ HIVE_OBJECT_TYPE,
+ HIVE_PRINCIPAL_TYPE,
+ HIVE_RESOURCE_TYPE,
+
+ // Structs
+ HIVE_SERDE,
+ HIVE_ORDER,
+ HIVE_RESOURCEURI,
+
+ // Classes
+ HIVE_DB,
+ HIVE_STORAGEDESC,
+ HIVE_TABLE,
+ HIVE_COLUMN,
+ HIVE_PARTITION,
+ HIVE_INDEX,
+ HIVE_ROLE,
+ HIVE_TYPE,
+ HIVE_PROCESS,
+ // HIVE_VIEW,
+ ;
+
+ public String getName() {
+ return name().toLowerCase();
+ }
+}
[13/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DataTypes.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DataTypes.java
deleted file mode 100755
index 3f94738..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DataTypes.java
+++ /dev/null
@@ -1,660 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.text.ParseException;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.Map;
-
-public class DataTypes {
-
- public static BooleanType BOOLEAN_TYPE = new BooleanType();
- public static ByteType BYTE_TYPE = new ByteType();
- public static ShortType SHORT_TYPE = new ShortType();
- public static IntType INT_TYPE = new IntType();
- public static LongType LONG_TYPE = new LongType();
- public static FloatType FLOAT_TYPE = new FloatType();
- public static DoubleType DOUBLE_TYPE = new DoubleType();
- public static BigIntegerType BIGINTEGER_TYPE = new BigIntegerType();
- public static BigDecimalType BIGDECIMAL_TYPE = new BigDecimalType();
- public static DateType DATE_TYPE = new DateType();
- public static StringType STRING_TYPE = new StringType();
- static String ARRAY_TYPE_PREFIX = "array<";
- static String ARRAY_TYPE_SUFFIX = ">";
- static String MAP_TYPE_PREFIX = "map<";
- static String MAP_TYPE_SUFFIX = ">";
-
- public static String arrayTypeName(String elemTypeName) {
- return String.format("%s%s%s", ARRAY_TYPE_PREFIX, elemTypeName, ARRAY_TYPE_SUFFIX);
- }
-
- public static String arrayTypeName(IDataType elemType) {
- return arrayTypeName(elemType.getName());
- }
-
- public static String mapTypeName(String keyTypeName, String valueTypeName) {
- return String.format("%s%s,%s%s", MAP_TYPE_PREFIX,
- keyTypeName, valueTypeName, MAP_TYPE_SUFFIX);
- }
-
- public static String mapTypeName(IDataType keyType, IDataType valueType) {
- return mapTypeName(keyType.getName(), valueType.getName());
- }
-
- public static enum TypeCategory {
- PRIMITIVE,
- ENUM,
- ARRAY,
- MAP,
- STRUCT,
- TRAIT,
- CLASS
- }
-
- public static abstract class PrimitiveType<T> extends AbstractDataType<T> {
- @Override
- public TypeCategory getTypeCategory() {
- return TypeCategory.PRIMITIVE;
- }
-
- public abstract T nullValue();
-
- }
-
- public static class BooleanType extends PrimitiveType<Boolean> {
-
- private static final String name = "boolean".intern();
-
- private BooleanType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Boolean convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Boolean) {
- return (Boolean) val;
- } else if (val instanceof String) {
- return Boolean.parseBoolean((String) val);
- } else if (val instanceof Number) {
- return ((Number) val).intValue() != 0;
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public Boolean nullValue() {
- return Boolean.FALSE;
- }
- }
-
- public static class ByteType extends PrimitiveType<Byte> {
-
- private static final String name = "byte".intern();
-
- private ByteType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Byte convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Byte) {
- return (Byte) val;
- } else if (val instanceof String) {
- return Byte.parseByte((String) val);
- } else if (val instanceof Number) {
- return ((Number) val).byteValue();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public Byte nullValue() {
- return 0;
- }
- }
-
- public static class ShortType extends PrimitiveType<Short> {
-
- private static final String name = "short".intern();
-
- private ShortType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Short convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Short) {
- return (Short) val;
- } else if (val instanceof String) {
- return Short.parseShort((String) val);
- } else if (val instanceof Number) {
- return ((Number) val).shortValue();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public Short nullValue() {
- return 0;
- }
- }
-
- public static class IntType extends PrimitiveType<Integer> {
-
- private static final String name = "int".intern();
-
- private IntType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Integer convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Integer) {
- return (Integer) val;
- } else if (val instanceof String) {
- return Integer.parseInt((String) val);
- } else if (val instanceof Number) {
- return ((Number) val).intValue();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public Integer nullValue() {
- return 0;
- }
- }
-
- public static class LongType extends PrimitiveType<Long> {
-
- private static final String name = "long".intern();
-
- private LongType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Long convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Long) {
- return (Long) val;
- } else if (val instanceof String) {
- return Long.parseLong((String) val);
- } else if (val instanceof Number) {
- return ((Number) val).longValue();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public Long nullValue() {
- return 0l;
- }
- }
-
- public static class FloatType extends PrimitiveType<Float> {
-
- private static final String name = "float".intern();
-
- private FloatType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Float convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Float) {
- return (Float) val;
- } else if (val instanceof String) {
- return Float.parseFloat((String) val);
- } else if (val instanceof Number) {
- return ((Number) val).floatValue();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public Float nullValue() {
- return 0.0f;
- }
- }
-
- public static class DoubleType extends PrimitiveType<Double> {
-
- private static final String name = "double".intern();
-
- private DoubleType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Double convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Double) {
- return (Double) val;
- } else if (val instanceof String) {
- return Double.parseDouble((String) val);
- } else if (val instanceof Number) {
- return ((Number) val).doubleValue();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public Double nullValue() {
- return 0.0;
- }
- }
-
- public static class BigIntegerType extends PrimitiveType<BigInteger> {
-
- private static final String name = "biginteger".intern();
-
- private BigIntegerType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public BigInteger convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof BigInteger) {
- return (BigInteger) val;
- } else if (val instanceof String) {
- try {
- return new BigInteger((String) val);
- } catch (NumberFormatException ne) {
- throw new ValueConversionException(this, val, ne);
- }
- } else if (val instanceof Number) {
- return BigInteger.valueOf(((Number) val).longValue());
- } else if (val instanceof BigDecimal) {
- return ((BigDecimal) val).toBigInteger();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public BigInteger nullValue() {
- return null;
- }
- }
-
- public static class BigDecimalType extends PrimitiveType<BigDecimal> {
-
- private static final String name = "bigdecimal".intern();
-
- private BigDecimalType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public BigDecimal convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof BigDecimal) {
- return (BigDecimal) val;
- } else if (val instanceof String) {
- try {
- return new BigDecimal((String) val);
- } catch (NumberFormatException ne) {
- throw new ValueConversionException(this, val, ne);
- }
- } else if (val instanceof Number) {
- return new BigDecimal(((Number) val).doubleValue());
- } else if (val instanceof BigInteger) {
- return new BigDecimal((BigInteger) val);
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- public BigDecimal nullValue() {
- return null;
- }
- }
-
- public static class DateType extends PrimitiveType<Date> {
-
- private static final String name = "date".intern();
-
- private DateType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public Date convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- if (val instanceof Date) {
- return (Date) val;
- } else if (val instanceof String) {
- try {
- return TypeSystem.getInstance().getDateFormat().parse((String) val);
- } catch (ParseException ne) {
- throw new ValueConversionException(this, val, ne);
- }
- } else if (val instanceof Number) {
- return new Date(((Number) val).longValue());
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- return convertNull(m);
- }
-
- @Override
- public void output(Date val, Appendable buf, String prefix) throws MetadataException {
- TypeUtils.outputVal(val == null ? "<null>" :
- TypeSystem.getInstance().getDateFormat().format(val), buf, prefix);
- }
-
- public Date nullValue() {
- return null;
- }
- }
-
- public static class StringType extends PrimitiveType<String> {
-
- private static final String name = "string".intern();
-
- private StringType() {
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String convert(Object val, Multiplicity m) throws MetadataException {
- if (StringUtils.isNotBlank((CharSequence) val)) {
- return val.toString();
- }
- return convertNull(m);
- }
-
- public String nullValue() {
- return null;
- }
- }
-
- public static class ArrayType extends AbstractDataType<ImmutableCollection<?>> {
- private final String nm;
- private IDataType elemType;
-
- public ArrayType(IDataType elemType) {
- assert elemType != null;
- this.elemType = elemType;
- this.nm = arrayTypeName(elemType);
- }
-
- public IDataType getElemType() {
- return elemType;
- }
-
- protected void setElemType(IDataType elemType) {
- this.elemType = elemType;
- }
-
- @Override
- public String getName() {
- return nm;
- }
-
- @Override
- public ImmutableCollection<?> convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- Iterator it = null;
- if (val instanceof Collection) {
- it = ((Collection) val).iterator();
- } else if (val instanceof Iterable) {
- it = ((Iterable) val).iterator();
- } else if (val instanceof Iterator) {
- it = (Iterator) val;
- }
- if (it != null) {
- ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder()
- : ImmutableList.builder();
- while (it.hasNext()) {
- b.add(elemType.convert(it.next(),
- TypeSystem.getInstance().allowNullsInCollections()
- ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
- }
- return m.isUnique ? b.build().asList() : b.build();
- } else {
- try {
- return ImmutableList.of(elemType.convert(val,
- TypeSystem.getInstance().allowNullsInCollections()
- ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
- } catch (Exception e) {
- throw new ValueConversionException(this, val, e);
- }
- }
- }
- if (!m.nullAllowed()) {
- throw new ValueConversionException.NullConversionException(m);
- }
- return null;
- }
-
- public ImmutableCollection<?> mapIds(ImmutableCollection<?> val, Multiplicity m,
- Map<Id, Id> transientToNewIds)
- throws MetadataException {
-
- if (val == null || elemType.getTypeCategory() != TypeCategory.CLASS) {
- return val;
- }
- ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder()
- : ImmutableList.builder();
- Iterator it = val.iterator();
- while (it.hasNext()) {
- Object elem = it.next();
- if (elem instanceof IReferenceableInstance) {
- Id oldId = ((IReferenceableInstance) elem).getId();
- Id newId = transientToNewIds.get(oldId);
- b.add(newId == null ? oldId : newId);
- } else {
- b.add(elem);
- }
- }
- return b.build();
- }
-
- @Override
- public TypeCategory getTypeCategory() {
- return TypeCategory.ARRAY;
- }
- }
-
- public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> {
-
- private final String nm;
- private IDataType keyType;
- private IDataType valueType;
-
- public MapType(IDataType keyType, IDataType valueType) {
- assert keyType != null;
- assert valueType != null;
- this.keyType = keyType;
- this.valueType = valueType;
- this.nm = mapTypeName(keyType, valueType);
- }
-
- public IDataType getKeyType() {
- return keyType;
- }
-
- protected void setKeyType(IDataType keyType) {
- this.keyType = keyType;
- }
-
- public IDataType getValueType() {
- return valueType;
- }
-
- protected void setValueType(IDataType valueType) {
- this.keyType = valueType;
- }
-
- @Override
- public String getName() {
- return nm;
- }
-
- @Override
- public ImmutableMap<?, ?> convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- Iterator<Map.Entry> it = null;
- if (Map.class.isAssignableFrom(val.getClass())) {
- it = ((Map) val).entrySet().iterator();
- ImmutableMap.Builder b = ImmutableMap.builder();
- while (it.hasNext()) {
- Map.Entry e = it.next();
- b.put(keyType.convert(e.getKey(),
- TypeSystem.getInstance().allowNullsInCollections()
- ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED),
- valueType.convert(e.getValue(),
- TypeSystem.getInstance().allowNullsInCollections()
- ? Multiplicity.OPTIONAL : Multiplicity.REQUIRED));
- }
- return b.build();
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- if (!m.nullAllowed()) {
- throw new ValueConversionException.NullConversionException(m);
- }
- return null;
- }
-
- public ImmutableMap<?, ?> mapIds(ImmutableMap val, Multiplicity m,
- Map<Id, Id> transientToNewIds)
- throws MetadataException {
-
- if (val == null || (keyType.getTypeCategory() != TypeCategory.CLASS &&
- valueType.getTypeCategory() != TypeCategory.CLASS)) {
- return val;
- }
- ImmutableMap.Builder b = ImmutableMap.builder();
- Iterator<Map.Entry> it = val.entrySet().iterator();
- while (it.hasNext()) {
- Map.Entry elem = it.next();
- Object oldKey = elem.getKey();
- Object oldValue = elem.getValue();
- Object newKey = oldKey;
- Object newValue = oldValue;
-
- if (oldKey instanceof IReferenceableInstance) {
- Id oldId = ((IReferenceableInstance) oldKey).getId();
- Id newId = transientToNewIds.get(oldId);
- newKey = newId == null ? oldId : newId;
- }
-
- if (oldValue instanceof IReferenceableInstance) {
- Id oldId = ((IReferenceableInstance) oldValue).getId();
- Id newId = transientToNewIds.get(oldId);
- newValue = newId == null ? oldId : newId;
- }
-
- b.put(newKey, newValue);
- }
- return b.build();
- }
-
- @Override
- public TypeCategory getTypeCategory() {
- return TypeCategory.MAP;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DownCastFieldMapping.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DownCastFieldMapping.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DownCastFieldMapping.java
deleted file mode 100755
index dfd7a4f..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/DownCastFieldMapping.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.persistence.DownCastStructInstance;
-
-public class DownCastFieldMapping {
-
- public final ImmutableMap<String, String> fieldNameMap;
-
- protected DownCastFieldMapping(ImmutableMap<String, String> fieldNameMap) {
- this.fieldNameMap = fieldNameMap;
- }
-
- public void set(DownCastStructInstance s, String attrName, Object val)
- throws MetadataException {
-
- String mappedNm = fieldNameMap.get(attrName);
- if (mappedNm == null) {
- throw new ValueConversionException(s.getTypeName(), val, "Unknown field " + attrName);
- }
-
- s.backingInstance.set(mappedNm, val);
- }
-
- public Object get(DownCastStructInstance s, String attrName) throws MetadataException {
-
- String mappedNm = fieldNameMap.get(attrName);
- if (mappedNm == null) {
- throw new ValueConversionException(
- String.format("Unknown field %s for Struct %s", attrName, s.getTypeName()));
- }
- return s.backingInstance.get(mappedNm);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumType.java
deleted file mode 100755
index 45974e6..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumType.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import scala.math.BigInt;
-
-public class EnumType extends AbstractDataType<EnumValue> {
-
- public final TypeSystem typeSystem;
- public final String name;
- public final ImmutableMap<String, EnumValue> valueMap;
- public final ImmutableMap<Integer, EnumValue> ordinalMap;
-
- protected EnumType(TypeSystem typeSystem, String name, EnumValue... values) {
- this.typeSystem = typeSystem;
- this.name = name;
- ImmutableMap.Builder<String, EnumValue> b1 = new ImmutableMap.Builder();
- ImmutableMap.Builder<Integer, EnumValue> b2 = new ImmutableMap.Builder();
- for (EnumValue v : values) {
- b1.put(v.value, v);
- b2.put(v.ordinal, v);
- }
- valueMap = b1.build();
- ordinalMap = b2.build();
- }
-
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public EnumValue convert(Object val, Multiplicity m) throws MetadataException {
- if (val != null) {
- EnumValue e = null;
- if (val instanceof EnumValue) {
- e = valueMap.get(((EnumValue)val).value);
- } else if ( val instanceof Integer || val instanceof BigInt) {
- e = ordinalMap.get(val);
- } else if ( val instanceof String) {
- e = valueMap.get(val);
- } else if ( val instanceof Number ) {
- e = ordinalMap.get(((Number)val).intValue());
- }
-
- if (e == null) {
- throw new ValueConversionException(this, val);
- }
- return e;
- }
- return convertNull(m);
- }
-
- @Override
- public DataTypes.TypeCategory getTypeCategory() {
- return DataTypes.TypeCategory.ENUM;
- }
-
- public EnumValue fromOrdinal(int o) {
- return ordinalMap.get(o);
- }
-
- public EnumValue fromValue(String val) {
- return valueMap.get(val.trim());
- }
-
- public ImmutableCollection<EnumValue> values() {
- return valueMap.values();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumTypeDefinition.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumTypeDefinition.java
deleted file mode 100755
index 6347a7b..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumTypeDefinition.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.ParamChecker;
-
-import java.util.Arrays;
-
-public final class EnumTypeDefinition {
-
- public final String name;
- public final EnumValue[] enumValues;
-
- public EnumTypeDefinition(String name, EnumValue... enumValues) {
- this.name = ParamChecker.notEmpty(name, "Enum type name");
- this.enumValues = ParamChecker.notNullElements(enumValues, "Enum values");
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- EnumTypeDefinition that = (EnumTypeDefinition) o;
-
- if (!Arrays.equals(enumValues, that.enumValues)) return false;
- if (!name.equals(that.name)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = name.hashCode();
- result = 31 * result + Arrays.hashCode(enumValues);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumValue.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumValue.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumValue.java
deleted file mode 100755
index f9341eb..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/EnumValue.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.ParamChecker;
-
-public class EnumValue {
-
- public final String value;
- public final int ordinal;
-
- public EnumValue(String value, int ordinal) {
- this.value = ParamChecker.notEmpty(value, "Enum value");
- this.ordinal = ordinal;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- EnumValue enumValue = (EnumValue) o;
-
- if (ordinal != enumValue.ordinal) return false;
- if (!value.equals(enumValue.value)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = value.hashCode();
- result = 31 * result + ordinal;
- return result;
- }
-
- @Override
- public String toString() {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/FieldMapping.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/FieldMapping.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/FieldMapping.java
deleted file mode 100755
index 8faae94..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/FieldMapping.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-
-import java.util.Map;
-
-public class FieldMapping {
-
- public final Map<String, AttributeInfo> fields;
- public final Map<String, Integer> fieldPos;
- public final Map<String, Integer> fieldNullPos;
- public final int numBools;
- public final int numBytes;
- public final int numShorts;
- public final int numInts;
- public final int numLongs;
- public final int numFloats;
- public final int numDoubles;
- public final int numBigInts;
- public final int numBigDecimals;
- public final int numDates;
- public final int numStrings;
- public final int numArrays;
- public final int numMaps;
- public final int numStructs;
- public final int numReferenceables;
-
- public FieldMapping(Map<String, AttributeInfo> fields, Map<String, Integer> fieldPos,
- Map<String, Integer> fieldNullPos, int numBools, int numBytes,
- int numShorts,
- int numInts, int numLongs, int numFloats, int numDoubles, int numBigInts,
- int numBigDecimals,
- int numDates, int numStrings, int numArrays, int numMaps, int numStructs,
- int numReferenceables) {
- this.fields = fields;
- this.fieldPos = fieldPos;
- this.fieldNullPos = fieldNullPos;
- this.numBools = numBools;
- this.numBytes = numBytes;
- this.numShorts = numShorts;
- this.numInts = numInts;
- this.numLongs = numLongs;
- this.numFloats = numFloats;
- this.numDoubles = numDoubles;
- this.numBigInts = numBigInts;
- this.numBigDecimals = numBigDecimals;
- this.numDates = numDates;
- this.numStrings = numStrings;
- this.numArrays = numArrays;
- this.numMaps = numMaps;
- this.numStructs = numStructs;
- this.numReferenceables = numReferenceables;
- }
-
- protected void outputFields(IStruct s, Appendable buf, String fieldPrefix)
- throws MetadataException {
- for (Map.Entry<String, AttributeInfo> e : fields.entrySet()) {
- String attrName = e.getKey();
- AttributeInfo i = e.getValue();
- Object aVal = s.get(attrName);
- TypeUtils.outputVal(attrName + " : ", buf, fieldPrefix);
- if (aVal != null && aVal instanceof Id) {
- TypeUtils.outputVal(aVal.toString(), buf, "");
- } else {
- i.dataType().output(aVal, buf, fieldPrefix);
- }
- TypeUtils.outputVal("\n", buf, "");
- }
- }
-
- public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
- if (s == null) {
- TypeUtils.outputVal("<null>\n", buf, "");
- return;
- }
- TypeUtils.outputVal("{", buf, prefix);
-
- TypeUtils.outputVal("\n", buf, "");
- String fieldPrefix = prefix + "\t";
-
- outputFields(s, buf, fieldPrefix);
-
- TypeUtils.outputVal("}", buf, prefix);
- }
-
- public void output(IReferenceableInstance s, Appendable buf, String prefix)
- throws MetadataException {
- if (s == null) {
- TypeUtils.outputVal("<null>\n", buf, "");
- return;
- }
- TypeUtils.outputVal("{", buf, prefix);
-
- TypeUtils.outputVal("\n", buf, "");
- String fieldPrefix = prefix + "\t";
-
- TypeUtils.outputVal("id : ", buf, fieldPrefix);
- TypeUtils.outputVal(s.getId().toString(), buf, "");
- TypeUtils.outputVal("\n", buf, "");
-
- outputFields(s, buf, fieldPrefix);
-
- TypeSystem ts = TypeSystem.getInstance();
-
- for (String sT : s.getTraits()) {
- TraitType tt = ts.getDataType(TraitType.class, sT);
- TypeUtils.outputVal(sT + " : ", buf, fieldPrefix);
- tt.output(s.getTrait(sT), buf, fieldPrefix);
- }
-
- TypeUtils.outputVal("}", buf, prefix);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalType.java
deleted file mode 100755
index 7fff806..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalType.java
+++ /dev/null
@@ -1,500 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.persistence.DownCastStructInstance;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils.Pair;
-
-/**
- * Represents a Type that can have SuperTypes. An Instance of the HierarchicalType can be
- * downcast to a SuperType.
- * @param <ST> the Type of the SuperType. TraitTypes have TraitTypes as SuperTypes, ClassTypes
- * have ClassTypes
- * as SuperTypes.
- * @param <T> the class of the Instance of this DataType.
- */
-public abstract class HierarchicalType<ST extends HierarchicalType, T> extends AbstractDataType<T>
- implements Comparable<ST> {
-
- public final TypeSystem typeSystem;
- public final Class<ST> superTypeClass;
- public final String name;
- public final FieldMapping fieldMapping;
- public final int numFields;
- public final ImmutableList<String> superTypes;
- public final ImmutableList<AttributeInfo> immediateAttrs;
- public final ImmutableMap<String, String> attributeNameToType;
- protected ImmutableMap<String, List<Path>> superTypePaths;
- protected ImmutableMap<String, Path> pathNameToPathMap;
-
- /**
- * Used when creating a Type, to support recursive Structs.
- */
- HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass,
- String name, ImmutableList<String> superTypes, int numFields) {
- this.typeSystem = typeSystem;
- this.superTypeClass = superTypeClass;
- this.name = name;
- this.fieldMapping = null;
- this.numFields = numFields;
- this.superTypes = superTypes;
- this.immediateAttrs = ImmutableList.of();
- this.attributeNameToType = null;
- }
-
- HierarchicalType(TypeSystem typeSystem, Class<ST> superTypeClass,
- String name, ImmutableList<String> superTypes, AttributeInfo... fields)
- throws MetadataException {
- this.typeSystem = typeSystem;
- this.superTypeClass = superTypeClass;
- this.name = name;
- Pair<FieldMapping, ImmutableMap<String, String>> p = constructFieldMapping(superTypes,
- fields);
- this.fieldMapping = p.left;
- this.attributeNameToType = p.right;
- this.numFields = this.fieldMapping.fields.size();
- this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes;
- this.immediateAttrs = ImmutableList.copyOf(fields);
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- public FieldMapping fieldMapping() {
- return fieldMapping;
- }
-
- /**
- * Given type must be a SubType of this type.
- * @param typeName
- * @throws MetadataException
- */
- public boolean isSubType(String typeName) throws MetadataException {
- HierarchicalType cType = typeSystem.getDataType(HierarchicalType.class, typeName);
- return (cType == this || cType.superTypePaths.containsKey(getName()));
- }
-
- protected void setupSuperTypesGraph()
- throws MetadataException {
- setupSuperTypesGraph(superTypes);
- }
-
- private void setupSuperTypesGraph(ImmutableList<String> superTypes)
- throws MetadataException {
- Map<String, List<Path>> superTypePaths = new HashMap<String, List<Path>>();
- Map<String, Path> pathNameToPathMap = new HashMap<String, Path>();
- Queue<Path> queue = new LinkedList<Path>();
- queue.add(new Node(getName()));
- while (!queue.isEmpty()) {
- Path currentPath = queue.poll();
-
- ST superType = currentPath.typeName == getName() ? (ST) this :
- (ST) typeSystem.getDataType(superTypeClass, currentPath.typeName);
-
- pathNameToPathMap.put(currentPath.pathName, currentPath);
- if (superType != this) {
- List<Path> typePaths = superTypePaths.get(superType.getName());
- if (typePaths == null) {
- typePaths = new ArrayList<Path>();
- superTypePaths.put(superType.getName(), typePaths);
- }
- typePaths.add(currentPath);
- }
-
- ImmutableList<String> sTs = superType == this ? superTypes : superType.superTypes;
-
- if (sTs != null) {
- for (String sT : sTs) {
- queue.add(new Path(sT, currentPath));
- }
- }
- }
-
- this.superTypePaths = ImmutableMap.copyOf(superTypePaths);
- this.pathNameToPathMap = ImmutableMap.copyOf(pathNameToPathMap);
-
- }
-
- protected Pair<FieldMapping, ImmutableMap<String, String>> constructFieldMapping(ImmutableList<String> superTypes,
- AttributeInfo... fields)
- throws MetadataException {
-
- Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
- Map<String, Integer> fieldPos = new HashMap<String, Integer>();
- Map<String, Integer> fieldNullPos = new HashMap<String, Integer>();
- Map<String, String> attributeNameToType = new HashMap<>();
-
- int numBools = 0;
- int numBytes = 0;
- int numShorts = 0;
- int numInts = 0;
- int numLongs = 0;
- int numFloats = 0;
- int numDoubles = 0;
- int numBigInts = 0;
- int numBigDecimals = 0;
- int numDates = 0;
- int numStrings = 0;
- int numArrays = 0;
- int numMaps = 0;
- int numStructs = 0;
- int numReferenceables = 0;
-
- setupSuperTypesGraph(superTypes);
-
- Iterator<Path> pathItr = pathIterator();
- while (pathItr.hasNext()) {
- Path currentPath = pathItr.next();
-
- ST superType = currentPath.typeName == getName() ? (ST) this :
- (ST) typeSystem.getDataType(superTypeClass, currentPath.typeName);
-
- ImmutableList<AttributeInfo> superTypeFields = superType == this ?
- ImmutableList.copyOf(fields) : superType.immediateAttrs;
-
- Set<String> immediateFields = new HashSet<String>();
-
- for (AttributeInfo i : superTypeFields) {
- if (superType == this) {
- if (immediateFields.contains(i.name)) {
- throw new MetadataException(
- String.format(
- "Struct defintion cannot contain multiple fields with the" +
- " same name %s",
- i.name));
- }
- immediateFields.add(i.name);
- }
-
- String attrName = i.name;
- if (fieldsMap.containsKey(attrName)) {
- attrName = currentPath.addOverrideAttr(attrName);
- }
- attributeNameToType.put(attrName, superType.getName());
-
- fieldsMap.put(attrName, i);
- fieldNullPos.put(attrName, fieldNullPos.size());
- if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
- fieldPos.put(attrName, numBools);
- numBools++;
- } else if (i.dataType() == DataTypes.BYTE_TYPE) {
- fieldPos.put(attrName, numBytes);
- numBytes++;
- } else if (i.dataType() == DataTypes.SHORT_TYPE) {
- fieldPos.put(attrName, numShorts);
- numShorts++;
- } else if (i.dataType() == DataTypes.INT_TYPE) {
- fieldPos.put(attrName, numInts);
- numInts++;
- } else if (i.dataType() == DataTypes.LONG_TYPE) {
- fieldPos.put(attrName, numLongs);
- numLongs++;
- } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
- fieldPos.put(attrName, numFloats);
- numFloats++;
- } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
- fieldPos.put(attrName, numDoubles);
- numDoubles++;
- } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
- fieldPos.put(attrName, numBigInts);
- numBigInts++;
- } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- fieldPos.put(attrName, numBigDecimals);
- numBigDecimals++;
- } else if (i.dataType() == DataTypes.DATE_TYPE) {
- fieldPos.put(attrName, numDates);
- numDates++;
- } else if (i.dataType() == DataTypes.STRING_TYPE) {
- fieldPos.put(attrName, numStrings);
- numStrings++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
- fieldPos.put(i.name, numInts);
- numInts++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- fieldPos.put(attrName, numArrays);
- numArrays++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
- fieldPos.put(attrName, numMaps);
- numMaps++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
- i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
- fieldPos.put(attrName, numStructs);
- numStructs++;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- fieldPos.put(attrName, numReferenceables);
- numReferenceables++;
- } else {
- throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
- }
- }
- }
-
- this.superTypePaths = ImmutableMap.copyOf(superTypePaths);
- this.pathNameToPathMap = ImmutableMap.copyOf(pathNameToPathMap);
-
- FieldMapping fm = new FieldMapping(fieldsMap,
- fieldPos,
- fieldNullPos,
- numBools,
- numBytes,
- numShorts,
- numInts,
- numLongs,
- numFloats,
- numDoubles,
- numBigInts,
- numBigDecimals,
- numDates,
- numStrings,
- numArrays,
- numMaps,
- numStructs,
- numReferenceables);
-
- return new Pair(fm, ImmutableMap.copyOf(attributeNameToType));
- }
-
- public IStruct castAs(IStruct s, String superTypeName) throws MetadataException {
-
- if (!superTypePaths.containsKey(superTypeName)) {
- throw new MetadataException(
- String.format("Cannot downcast to %s from type %s", superTypeName, getName()));
- }
-
- if (s != null) {
- if (s.getTypeName() != getName()) {
- throw new MetadataException(
- String.format("Downcast called on wrong type %s, instance type is %s",
- getName(), s.getTypeName()));
- }
-
- List<Path> pathToSuper = superTypePaths.get(superTypeName);
- if (pathToSuper.size() > 1) {
- throw new MetadataException(
- String.format(
- "Cannot downcast called to %s, from %s: there are multiple paths " +
- "to SuperType",
- superTypeName, getName()));
- }
-
- ST superType = (ST) typeSystem.getDataType(superTypeClass, superTypeName);
- Map<String, String> downCastMap = superType
- .constructDowncastFieldMap(this, pathToSuper.get(0));
- return new DownCastStructInstance(superTypeName,
- new DownCastFieldMapping(ImmutableMap.copyOf(downCastMap)),
- s);
- }
-
- return null;
- }
-
- public ST getDefinedType(String attrName) throws MetadataException {
- if (!attributeNameToType.containsKey(attrName)) {
- throw new MetadataException(String.format("Unknown attribute %s in type %s", attrName, getName()));
- }
- return typeSystem.getDataType(superTypeClass, attributeNameToType.get(attrName));
- }
-
- public String getDefinedTypeName(String attrName) throws MetadataException {
- return getDefinedType(attrName).getName();
- }
-
- public String getQualifiedName(String attrName) throws MetadataException {
- String attrTypeName = getDefinedTypeName(attrName);
- return attrName.contains(".") ? attrName : String.format("%s.%s", attrTypeName, attrName);
- }
-
- protected Map<String, String> constructDowncastFieldMap(ST subType, Path pathToSubType) {
-
- String pathToSubTypeName = pathToSubType.pathAfterThis;
- /*
- * the downcastMap;
- */
- Map<String, String> dCMap = new HashMap<String, String>();
- Iterator<Path> itr = pathIterator();
- while (itr.hasNext()) {
- Path p = itr.next();
- Path pInSubType = (Path) subType.pathNameToPathMap
- .get(p.pathName + "." + pathToSubTypeName);
-
- if (pInSubType.hiddenAttributeMap != null) {
- for (Map.Entry<String, String> e : pInSubType.hiddenAttributeMap.entrySet()) {
- String mappedInThisType =
- p.hiddenAttributeMap != null ? p.hiddenAttributeMap.get(e.getKey())
- : null;
- if (mappedInThisType == null) {
- dCMap.put(e.getKey(), e.getValue());
- } else {
- dCMap.put(mappedInThisType, e.getValue());
- }
- }
- }
- }
- return dCMap;
- }
-
- @Override
- public int compareTo(ST o) {
- String oName = o.getName();
- if (superTypes.contains(oName)) {
- return 1;
- } else if (o.superTypes.contains(getName())) {
- return -1;
- } else {
- return getName().compareTo(oName);
- }
- }
-
- public Set<String> getAllSuperTypeNames() {
- return superTypePaths.keySet();
- }
-
- public Iterator<Path> pathIterator() {
- return new PathItr();
- }
-
- static class Path {
- public final String typeName;
- public final String pathName;
- public final String pathAfterThis;
- private final Path subTypePath;
- /*
- * name mapping for attributes hidden by a SubType.
- */
- Map<String, String> hiddenAttributeMap;
-
- Path(String typeName, Path childPath) throws MetadataException {
- this.typeName = typeName;
- this.subTypePath = childPath;
- if (childPath.contains(typeName)) {
- throw new CyclicTypeDefinition(this);
- }
- pathName = String.format("%s.%s", typeName, childPath.pathName);
- pathAfterThis = childPath.pathName;
- }
-
- Path(String typeName) {
- assert getClass() == Node.class;
- this.typeName = typeName;
- this.subTypePath = null;
- pathName = typeName;
- pathAfterThis = null;
- }
-
- public boolean contains(String typeName) {
- return this.typeName.equals(typeName) ||
- (subTypePath != null && subTypePath.contains(typeName));
- }
-
- public String pathString(String nodeSep) {
-
- StringBuilder b = new StringBuilder();
- Path p = this;
-
- while (p != null) {
- b.append(p.typeName);
- p = p.subTypePath;
- if (p != null) {
- b.append(nodeSep);
- }
- }
- return b.toString();
- }
-
- String addOverrideAttr(String name) {
- hiddenAttributeMap = hiddenAttributeMap == null ? new HashMap<String, String>()
- : hiddenAttributeMap;
- String oName = pathName + "." + name;
- hiddenAttributeMap.put(name, oName);
- return oName;
- }
- }
-
- static class Node extends Path {
- Node(String typeName) {
- super(typeName);
- }
- }
-
- static class CyclicTypeDefinition extends MetadataException {
-
- CyclicTypeDefinition(Path p) {
- super(String.format("Cycle in Type Definition %s", p.pathString(" -> ")));
- }
- }
-
- class PathItr implements Iterator<Path> {
-
- Queue<Path> pathQueue;
-
- PathItr() {
- pathQueue = new LinkedList<Path>();
- pathQueue.add(pathNameToPathMap.get(getName()));
- }
-
- @Override
- public boolean hasNext() {
- return !pathQueue.isEmpty();
- }
-
- @Override
- public Path next() {
- Path p = pathQueue.poll();
- ST t = null;
- try {
- t = (ST) typeSystem.getDataType(superTypeClass, p.typeName);
- } catch (MetadataException me) {
- throw new RuntimeException(me);
- }
- if (t.superTypes != null) {
- ImmutableList<String> sTs = t.superTypes;
- for (String sT : sTs) {
- String nm = sT + "." + p.pathName;
- pathQueue.add(pathNameToPathMap.get(nm));
- }
- }
- return p;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalTypeDefinition.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalTypeDefinition.java
deleted file mode 100755
index 08c8a07..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/HierarchicalTypeDefinition.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.classification.InterfaceAudience;
-
-public class HierarchicalTypeDefinition<T extends HierarchicalType> extends StructTypeDefinition {
-
- public final ImmutableList<String> superTypes;
- public final String hierarchicalMetaTypeName;
-
- /**
- * Used for json deserialization only.
- * not intended public consumption
- * @param hierarchicalMetaTypeName
- * @param typeName
- * @param superTypes
- * @param attributeDefinitions
- * @throws ClassNotFoundException
- */
- @InterfaceAudience.Private
- public HierarchicalTypeDefinition(String hierarchicalMetaTypeName,
- String typeName, String[] superTypes,
- AttributeDefinition[] attributeDefinitions)
- throws ClassNotFoundException {
- this((Class<T>) Class.forName(hierarchicalMetaTypeName),
- typeName, ImmutableList.copyOf(superTypes), attributeDefinitions);
- }
-
- public HierarchicalTypeDefinition(Class<T> hierarchicalMetaType,
- String typeName, ImmutableList<String> superTypes,
- AttributeDefinition[] attributeDefinitions) {
- super(typeName, false, attributeDefinitions);
- hierarchicalMetaTypeName = hierarchicalMetaType.getName();
- this.superTypes = superTypes == null ? ImmutableList.<String>of() : superTypes;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
-
- HierarchicalTypeDefinition that = (HierarchicalTypeDefinition) o;
-
- if (!hierarchicalMetaTypeName.equals(that.hierarchicalMetaTypeName)) return false;
- if (!superTypes.equals(that.superTypes)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + superTypes.hashCode();
- result = 31 * result + hierarchicalMetaTypeName.hashCode();
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IConstructableType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IConstructableType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IConstructableType.java
deleted file mode 100755
index fb5fafb..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IConstructableType.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.ITypedInstance;
-
-import java.util.List;
-
-public interface IConstructableType<U, T extends ITypedInstance> extends IDataType<U> {
-
- T createInstance() throws MetadataException;
-
- FieldMapping fieldMapping();
-
- List<String> getNames(AttributeInfo info);
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IDataType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IDataType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IDataType.java
deleted file mode 100755
index bce2f9f..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/IDataType.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-public interface IDataType<T> {
- String getName();
-
- T convert(Object val, Multiplicity m) throws MetadataException;
-
- DataTypes.TypeCategory getTypeCategory();
-
- void output(T val, Appendable buf, String prefix) throws MetadataException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/Multiplicity.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/Multiplicity.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/Multiplicity.java
deleted file mode 100755
index c28e08a..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/Multiplicity.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-
-public final class Multiplicity {
-
- public static final Multiplicity OPTIONAL = new Multiplicity(0, 1, false);
- public static final Multiplicity REQUIRED = new Multiplicity(1, 1, false);
- public static final Multiplicity COLLECTION = new Multiplicity(1, Integer.MAX_VALUE, false);
- public static final Multiplicity SET = new Multiplicity(1, Integer.MAX_VALUE, true);
-
- public final int lower;
- public final int upper;
- public final boolean isUnique;
- public Multiplicity(int lower, int upper, boolean isUnique) {
- assert lower >= 0;
- assert upper >= 1;
- assert upper >= lower;
- this.lower = lower;
- this.upper = upper;
- this.isUnique = isUnique;
- }
-
- public boolean nullAllowed() {
- return lower == 0;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- Multiplicity that = (Multiplicity) o;
-
- if (isUnique != that.isUnique) return false;
- if (lower != that.lower) return false;
- if (upper != that.upper) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = lower;
- result = 31 * result + upper;
- result = 31 * result + (isUnique ? 1 : 0);
- return result;
- }
-
- @Override
- public String toString() {
- return "Multiplicity{" +
- "lower=" + lower +
- ", upper=" + upper +
- ", isUnique=" + isUnique +
- '}';
- }
-
- public String toJson() throws JSONException {
- JSONObject json = new JSONObject();
- json.put("lower", lower);
- json.put("upper", upper);
- json.put("isUnique", isUnique);
- return json.toString();
- }
-
- public static Multiplicity fromJson(String jsonStr) throws JSONException {
- JSONObject json = new JSONObject(jsonStr);
- return new Multiplicity(json.getInt("lower"), json.getInt("upper"), json.getBoolean("isUnique"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphTraversal.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphTraversal.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphTraversal.java
deleted file mode 100755
index dea469f..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphTraversal.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-
-public class ObjectGraphTraversal implements Iterator<ObjectGraphTraversal.InstanceTuple> {
-
- final Queue<InstanceTuple> queue;
- final TypeSystem typeSystem;
- Set<Id> processedIds;
-
- public ObjectGraphTraversal(TypeSystem typeSystem, IReferenceableInstance start)
- throws MetadataException {
- this.typeSystem = typeSystem;
- queue = new LinkedList<InstanceTuple>();
- processedIds = new HashSet<Id>();
- processReferenceableInstance(start);
- }
-
- void processValue(IDataType dT, Object val) throws MetadataException {
- if (val != null) {
- if (dT.getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- IDataType elemType = ((DataTypes.ArrayType) dT).getElemType();
- processCollection(elemType, val);
- } else if (dT.getTypeCategory() == DataTypes.TypeCategory.MAP) {
- IDataType keyType = ((DataTypes.MapType) dT).getKeyType();
- IDataType valueType = ((DataTypes.MapType) dT).getKeyType();
- processMap(keyType, valueType, val);
- } else if (dT.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
- dT.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
- processStruct(val);
- } else if (dT.getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- processReferenceableInstance(val);
- }
- }
- }
-
- void processMap(IDataType keyType, IDataType valueType, Object val) throws MetadataException {
- if (keyType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE &&
- valueType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
- return;
- }
-
- if (val != null) {
- Iterator<Map.Entry> it = null;
- if (Map.class.isAssignableFrom(val.getClass())) {
- it = ((Map) val).entrySet().iterator();
- ImmutableMap.Builder b = ImmutableMap.builder();
- while (it.hasNext()) {
- Map.Entry e = it.next();
- processValue(keyType, e.getKey());
- processValue(valueType, e.getValue());
- }
- }
- }
- }
-
- void processCollection(IDataType elemType, Object val) throws MetadataException {
-
- if (elemType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
- return;
- }
-
- if (val != null) {
- Iterator it = null;
- if (val instanceof Collection) {
- it = ((Collection) val).iterator();
- } else if (val instanceof Iterable) {
- it = ((Iterable) val).iterator();
- } else if (val instanceof Iterator) {
- it = (Iterator) val;
- }
- if (it != null) {
- DataTypes.TypeCategory elemCategory = elemType.getTypeCategory();
- while (it.hasNext()) {
- Object elem = it.next();
- processValue(elemType, elem);
- }
- }
- }
- }
-
- void processStruct(Object val) throws MetadataException {
-
- if (val == null || !(val instanceof IStruct)) {
- return;
- }
-
- IStruct i = (IStruct) val;
-
- IConstructableType type = typeSystem.getDataType(IConstructableType.class, i.getTypeName());
-
- for (Map.Entry<String, AttributeInfo> e : type.fieldMapping().fields.entrySet()) {
- AttributeInfo aInfo = e.getValue();
- String attrName = e.getKey();
- if (aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE) {
- processValue(aInfo.dataType(), i.get(attrName));
- }
- }
- }
-
- void processReferenceableInstance(Object val) throws MetadataException {
-
- if (val == null || !(val instanceof IReferenceableInstance || val instanceof Id)) {
- return;
- }
-
- if (val instanceof Id) {
- Id id = (Id) val;
- if (id.isUnassigned()) {
- add(id, null);
- }
- return;
- }
-
- IReferenceableInstance ref = (IReferenceableInstance) val;
- Id id = ref.getId();
- if (id.isUnassigned()) {
- add(id, ref);
- if (!processedIds.contains(id)) {
- processedIds.add(id);
- processStruct(val);
-
- ImmutableList<String> traits = ref.getTraits();
- for (String trait : traits) {
- processStruct(ref.getTrait(trait));
- }
- }
- }
- }
-
- void add(Id id, IReferenceableInstance ref) {
- queue.add(new InstanceTuple(id, ref));
- }
-
-
- @Override
- public boolean hasNext() {
- return !queue.isEmpty();
- }
-
- @Override
- public InstanceTuple next() {
- try {
- InstanceTuple t = queue.poll();
- processReferenceableInstance(t.instance);
- return t;
- } catch (MetadataException me) {
- throw new RuntimeException(me);
- }
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- public static class InstanceTuple {
- public final Id id;
- public final IReferenceableInstance instance;
-
- public InstanceTuple(Id id, IReferenceableInstance instance) {
- this.id = id;
- this.instance = instance;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphWalker.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphWalker.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphWalker.java
deleted file mode 100755
index b3f2062..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ObjectGraphWalker.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-
-/**
- * Given a IReferenceableInstance, a Walker will traverse the Object Graph
- * reachable form the instance. It will invoke the process call on the provided NodeProcessor
- * for each non-primitive attribute (Structs, Traits, References, Arrays of Non-Primitives, Maps
- * of Non-Primitives)
- */
-public class ObjectGraphWalker {
-
- final Queue<IReferenceableInstance> queue;
- final TypeSystem typeSystem;
- final NodeProcessor nodeProcessor;
- Set<Id> processedIds;
-
- public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor)
- throws MetadataException {
- this(typeSystem, nodeProcessor, (IReferenceableInstance) null);
- }
-
- public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor,
- IReferenceableInstance start)
- throws MetadataException {
- this.typeSystem = typeSystem;
- this.nodeProcessor = nodeProcessor;
- queue = new LinkedList<IReferenceableInstance>();
- processedIds = new HashSet<Id>();
- if (start != null) {
- visitReferenceableInstance(start);
- }
- }
-
- public ObjectGraphWalker(TypeSystem typeSystem, NodeProcessor nodeProcessor,
- List<? extends IReferenceableInstance> roots)
- throws MetadataException {
- this.typeSystem = typeSystem;
- this.nodeProcessor = nodeProcessor;
- queue = new LinkedList<IReferenceableInstance>();
- processedIds = new HashSet<Id>();
- for (IReferenceableInstance r : roots) {
- visitReferenceableInstance(r);
- }
- }
-
- public void walk() throws MetadataException {
- while (!queue.isEmpty()) {
- IReferenceableInstance r = queue.poll();
- processReferenceableInstance(r);
- }
- }
-
- public void addRoot(IReferenceableInstance root) {
- visitReferenceableInstance(root);
- }
-
- void traverseValue(IDataType dT, Object val) throws MetadataException {
- if (val != null) {
- if (dT.getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- IDataType elemType = ((DataTypes.ArrayType) dT).getElemType();
- visitCollection(elemType, val);
- } else if (dT.getTypeCategory() == DataTypes.TypeCategory.MAP) {
- IDataType keyType = ((DataTypes.MapType) dT).getKeyType();
- IDataType valueType = ((DataTypes.MapType) dT).getKeyType();
- visitMap(keyType, valueType, val);
- } else if (dT.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
- dT.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
- visitStruct(val);
- } else if (dT.getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- visitReferenceableInstance(val);
- }
- }
- }
-
- void visitMap(IDataType keyType, IDataType valueType, Object val) throws MetadataException {
- if (keyType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE &&
- valueType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
- return;
- }
-
- if (val != null) {
- Iterator<Map.Entry> it = null;
- if (Map.class.isAssignableFrom(val.getClass())) {
- it = ((Map) val).entrySet().iterator();
- ImmutableMap.Builder b = ImmutableMap.builder();
- while (it.hasNext()) {
- Map.Entry e = it.next();
- traverseValue(keyType, e.getKey());
- traverseValue(valueType, e.getValue());
- }
- }
- }
- }
-
- void visitCollection(IDataType elemType, Object val) throws MetadataException {
-
- if (elemType.getTypeCategory() == DataTypes.TypeCategory.PRIMITIVE) {
- return;
- }
-
- if (val != null) {
- Iterator it = null;
- if (val instanceof Collection) {
- it = ((Collection) val).iterator();
- } else if (val instanceof Iterable) {
- it = ((Iterable) val).iterator();
- } else if (val instanceof Iterator) {
- it = (Iterator) val;
- }
- if (it != null) {
- DataTypes.TypeCategory elemCategory = elemType.getTypeCategory();
- while (it.hasNext()) {
- Object elem = it.next();
- traverseValue(elemType, elem);
- }
- }
- }
- }
-
- void visitStruct(Object val) throws MetadataException {
-
- if (val == null || !(val instanceof IStruct)) {
- return;
- }
-
- IStruct i = (IStruct) val;
-
- IConstructableType type = typeSystem.getDataType(IConstructableType.class, i.getTypeName());
-
- for (Map.Entry<String, AttributeInfo> e : type.fieldMapping().fields.entrySet()) {
- AttributeInfo aInfo = e.getValue();
- String attrName = e.getKey();
- if (aInfo.dataType().getTypeCategory() != DataTypes.TypeCategory.PRIMITIVE) {
- Object aVal = i.get(attrName);
- nodeProcessor.processNode(new Node(i, attrName, aInfo, aVal));
- traverseValue(aInfo.dataType(), aVal);
- }
- }
- }
-
- void visitReferenceableInstance(Object val) {
-
- if (val == null || !(val instanceof IReferenceableInstance)) {
- return;
- }
-
- IReferenceableInstance ref = (IReferenceableInstance) val;
-
- if (!processedIds.contains(ref.getId())) {
- processedIds.add(ref.getId());
- if (!(ref instanceof Id)) {
- queue.add(ref);
- }
- }
- }
-
- void processReferenceableInstance(IReferenceableInstance ref) throws MetadataException {
-
- nodeProcessor.processNode(new Node(ref, null, null, null));
- visitStruct(ref);
- ImmutableList<String> traits = ref.getTraits();
- for (String trait : traits) {
- visitStruct(ref.getTrait(trait));
- }
- }
-
- public static interface NodeProcessor {
-
- void processNode(Node nd) throws MetadataException;
- }
-
- /**
- * Represents a non-primitive value of an instance.
- */
- public static class Node {
- public final IStruct instance;
- public final String attributeName;
- public final AttributeInfo aInfo;
- public final Object value;
-
- public Node(IStruct instance, String attributeName, AttributeInfo aInfo, Object value) {
- this.instance = instance;
- this.attributeName = attributeName;
- this.aInfo = aInfo;
- this.value = value;
- }
- }
-}
[14/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Struct.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Struct.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Struct.java
deleted file mode 100755
index 18447a5..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/Struct.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem;
-
-import org.apache.hadoop.metadata.classification.InterfaceAudience;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class Struct implements IStruct {
-
- public final String typeName;
- private final Map<String, Object> values;
-
- public Struct(String typeName) {
- this.typeName = typeName;
- values = new HashMap<>();
- }
-
- @InterfaceAudience.Private
- public Struct(String typeName, Map<String, Object> values) {
- this(typeName);
- if (values != null) {
- this.values.putAll(values);
- }
- }
-
- @Override
- public String getTypeName() {
- return typeName;
- }
-
- @Override
- public Object get(String attrName) {
- return values.get(attrName);
- }
-
- @Override
- public void set(String attrName, Object value) {
- values.put(attrName, value);
- }
-
- @Override
- public Map<String, Object> getValuesMap() {
- return values;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/DownCastStructInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/DownCastStructInstance.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/DownCastStructInstance.java
deleted file mode 100755
index 57af152..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/DownCastStructInstance.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.persistence;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.types.DownCastFieldMapping;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class DownCastStructInstance implements IStruct {
-
- public final String typeName;
- public final DownCastFieldMapping fieldMapping;
- public final IStruct backingInstance;
-
- public DownCastStructInstance(String typeName, DownCastFieldMapping fieldMapping,
- IStruct backingInstance) {
- this.typeName = typeName;
- this.fieldMapping = fieldMapping;
- this.backingInstance = backingInstance;
- }
-
- @Override
- public String getTypeName() {
- return typeName;
- }
-
- @Override
- public Object get(String attrName) throws MetadataException {
- return fieldMapping.get(this, attrName);
- }
-
- @Override
- public void set(String attrName, Object val) throws MetadataException {
- fieldMapping.set(this, attrName, val);
- }
-
- /*
- * Use only for json serialization
- * @nonpublic
- */
- @Override
- public Map<String, Object> getValuesMap() throws MetadataException {
-
- Map<String,Object> m = new HashMap<>();
- for (String attr : fieldMapping.fieldNameMap.keySet()) {
- m.put(attr, get(attr));
- }
- return m;
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/Id.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/Id.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/Id.java
deleted file mode 100755
index ec524f1..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/Id.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.persistence;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.types.FieldMapping;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-import java.util.UUID;
-
-public class Id implements ITypedReferenceableInstance {
-
- public final String id;
- public final String className;
- public final int version;
-
- public Id(String id, int version, String className) {
- this.id = id;
- this.className = className;
- this.version = version;
- }
-
- public Id(long id, int version, String className) {
- this("" + id, version, className);
- }
-
- public Id(String className) {
- this("" + (-System.nanoTime()), 0, className);
- }
-
- public boolean isUnassigned() {
- try {
- long l = Long.parseLong(id);
- return l < 0;
- } catch (NumberFormatException ne) {
- return false;
- }
- }
-
- public boolean isAssigned() {
- try {
- UUID.fromString(id);
- } catch (IllegalArgumentException e) {
- return false;
- }
-
- return true;
- }
-
- public String toString() {
- return String
- .format("(type: %s, id: %s)", className, isUnassigned() ? "<unassigned>" : "" + id);
- }
-
- public String getClassName() {
- return className;
- }
-
- public int getVersion() {
- return version;
- }
-
- public String _getId() {
- return id;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- Id id1 = (Id) o;
-
- if (version != id1.version) return false;
- if (!className.equals(id1.className)) return false;
- if (!id.equals(id1.id)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = id.hashCode();
- result = 31 * result + className.hashCode();
- result = 31 * result + version;
- return result;
- }
-
- @Override
- public ImmutableList<String> getTraits() {
- return null;
- }
-
- @Override
- public Id getId() {
- return this;
- }
-
- @Override
- public IStruct getTrait(String typeName) {
- return null;
- }
-
- @Override
- public String getTypeName() {
- return className;
- }
-
- @Override
- public Object get(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- @Override
- public void set(String attrName, Object val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- @Override
- public FieldMapping fieldMapping() {
- return null;
- }
-
- @Override
- public Map<String, Object> getValuesMap() throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setNull(String attrName) throws MetadataException {
- set(attrName, null);
- }
-
- public boolean getBoolean(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public byte getByte(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public short getShort(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public int getInt(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public long getLong(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public float getFloat(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public double getDouble(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public BigInteger getBigInt(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public BigDecimal getBigDecimal(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public Date getDate(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public String getString(String attrName) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setBoolean(String attrName, boolean val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setByte(String attrName, byte val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setShort(String attrName, short val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setInt(String attrName, int val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setLong(String attrName, long val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setFloat(String attrName, float val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setDouble(String attrName, double val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setBigInt(String attrName, BigInteger val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setBigDecimal(String attrName, BigDecimal val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setDate(String attrName, Date val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-
- public void setString(String attrName, String val) throws MetadataException {
- throw new MetadataException("Get/Set not supported on an Id object");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/MapIds.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/MapIds.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/MapIds.java
deleted file mode 100755
index 2145857..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/MapIds.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.persistence;
-
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker;
-
-import java.util.Map;
-
-public class MapIds implements ObjectGraphWalker.NodeProcessor {
-
- final Map<Id, Id> idToNewIdMap;
-
- public MapIds(Map<Id, Id> idToNewIdMap) {
- this.idToNewIdMap = idToNewIdMap;
- }
-
- @Override
- public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
-
- IReferenceableInstance ref = null;
- Id id = null;
-
- if (nd.attributeName == null) {
- ref = (IReferenceableInstance) nd.instance;
- Id newId = idToNewIdMap.get(ref.getId());
- if (newId != null) {
- ((ReferenceableInstance) ref).replaceWithNewId(newId);
- }
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (nd.value != null && nd.value instanceof IReferenceableInstance) {
- Id oldId = ((IReferenceableInstance) nd.value).getId();
- Id newId = idToNewIdMap.get(oldId);
- /*
- * Replace Instances with Ids, irrespective of whether they map to newIds or not.
- */
- newId = newId == null ? oldId : newId;
- nd.instance.set(nd.attributeName, newId);
- }
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- DataTypes.ArrayType aT = (DataTypes.ArrayType) nd.aInfo.dataType();
- Object v = aT
- .mapIds((ImmutableCollection) nd.value, nd.aInfo.multiplicity, idToNewIdMap);
- nd.instance.set(nd.attributeName, v);
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
- DataTypes.MapType mT = (DataTypes.MapType) nd.aInfo.dataType();
- Object v = mT.mapIds((ImmutableMap) nd.value, nd.aInfo.multiplicity, idToNewIdMap);
- nd.instance.set(nd.attributeName, v);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/ReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/ReferenceableInstance.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/ReferenceableInstance.java
deleted file mode 100755
index ff5fc1d..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/ReferenceableInstance.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.persistence;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.types.FieldMapping;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-
-/*
- * @todo handle names prefixed by traitName.
- */
-public class ReferenceableInstance extends StructInstance implements ITypedReferenceableInstance {
-
- private final ImmutableMap<String, ITypedStruct> traits;
- private final ImmutableList<String> traitNames;
- private Id id;
-
-
- public ReferenceableInstance(Id id, String dataTypeName, FieldMapping fieldMapping,
- boolean[] nullFlags,
- boolean[] bools, byte[] bytes, short[] shorts, int[] ints,
- long[] longs,
- float[] floats, double[] doubles, BigDecimal[] bigDecimals,
- BigInteger[] bigIntegers, Date[] dates, String[] strings,
- ImmutableList<Object>[] arrays,
- ImmutableMap<Object, Object>[] maps,
- StructInstance[] structs,
- ReferenceableInstance[] referenceableInstances,
- Id[] ids,
- ImmutableMap<String, ITypedStruct> traits) {
- super(dataTypeName, fieldMapping, nullFlags, bools, bytes, shorts, ints, longs, floats,
- doubles, bigDecimals,
- bigIntegers, dates, strings, arrays, maps, structs, referenceableInstances, ids);
- this.id = id;
- this.traits = traits;
- ImmutableList.Builder<String> b = new ImmutableList.Builder<String>();
- for (String t : traits.keySet()) {
- b.add(t);
- }
- this.traitNames = b.build();
- }
-
- @Override
- public ImmutableList<String> getTraits() {
- return traitNames;
- }
-
- @Override
- public Id getId() {
- return id;
- }
-
- @Override
- public IStruct getTrait(String typeName) {
- return traits.get(typeName);
- }
-
- /**
- * @nopub
- * @param id
- */
- void replaceWithNewId(Id id) {
- this.id = id;
- }
-
- @Override
- public String toString() {
- try {
- StringBuilder buf = new StringBuilder();
- String prefix = "";
-
- fieldMapping.output(this, buf, prefix);
- return buf.toString();
-
- } catch (MetadataException me) {
- throw new RuntimeException(me);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/StructInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/StructInstance.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/StructInstance.java
deleted file mode 100755
index c622a47..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/persistence/StructInstance.java
+++ /dev/null
@@ -1,757 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.persistence;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumType;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.FieldMapping;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.ValueConversionException;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-public class StructInstance implements ITypedStruct {
- public final String dataTypeName;
- public final FieldMapping fieldMapping;
- public final boolean nullFlags[];
- public final boolean[] bools;
- public final byte[] bytes;
- public final short[] shorts;
- public final int[] ints;
- public final long[] longs;
- public final float[] floats;
- public final double[] doubles;
- public final BigDecimal[] bigDecimals;
- public final BigInteger[] bigIntegers;
- public final Date[] dates;
- public final String[] strings;
- public final ImmutableList<Object>[] arrays;
- public final ImmutableMap<Object, Object>[] maps;
- public final StructInstance[] structs;
- public final ReferenceableInstance[] referenceables;
- public final Id[] ids;
-
- public StructInstance(String dataTypeName, FieldMapping fieldMapping,
- boolean[] nullFlags, boolean[] bools, byte[] bytes, short[] shorts,
- int[] ints,
- long[] longs, float[] floats, double[] doubles,
- BigDecimal[] bigDecimals, BigInteger[] bigIntegers, Date[] dates,
- String[] strings,
- ImmutableList<Object>[] arrays, ImmutableMap<Object, Object>[] maps,
- StructInstance[] structs, ReferenceableInstance[] referenceables,
- Id[] ids) {
- assert dataTypeName != null;
- this.dataTypeName = dataTypeName;
- this.fieldMapping = fieldMapping;
- this.nullFlags = nullFlags;
- this.bools = bools;
- this.bytes = bytes;
- this.shorts = shorts;
- this.ints = ints;
- this.longs = longs;
- this.floats = floats;
- this.doubles = doubles;
- this.bigDecimals = bigDecimals;
- this.bigIntegers = bigIntegers;
- this.dates = dates;
- this.strings = strings;
- this.arrays = arrays;
- this.maps = maps;
- this.structs = structs;
- this.referenceables = referenceables;
- this.ids = ids;
-
- for (int i = 0; i < nullFlags.length; i++) {
- nullFlags[i] = true;
- }
- }
-
- @Override
- public String getTypeName() {
- return dataTypeName;
- }
-
- @Override
- public FieldMapping fieldMapping() {
- return fieldMapping;
- }
-
- public void set(String attrName, Object val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new ValueConversionException(getTypeName(), val, "Unknown field " + attrName);
- }
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
- Object cVal = null;
-
- if (val != null && val instanceof Id) {
- ClassType clsType =
- TypeSystem.getInstance().getDataType(ClassType.class, i.dataType().getName());
- clsType.validateId((Id) val);
- cVal = val;
- } else {
- cVal = i.dataType().convert(val, i.multiplicity);
- }
- if (cVal == null) {
- nullFlags[nullPos] = true;
- return;
- }
- nullFlags[nullPos] = false;
- if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
- bools[pos] = ((Boolean) cVal).booleanValue();
- } else if (i.dataType() == DataTypes.BYTE_TYPE) {
- bytes[pos] = ((Byte) cVal).byteValue();
- } else if (i.dataType() == DataTypes.SHORT_TYPE) {
- shorts[pos] = ((Short) cVal).shortValue();
- } else if (i.dataType() == DataTypes.INT_TYPE) {
- ints[pos] = ((Integer) cVal).intValue();
- } else if (i.dataType() == DataTypes.LONG_TYPE) {
- longs[pos] = ((Long) cVal).longValue();
- } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
- floats[pos] = ((Float) cVal).floatValue();
- } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
- doubles[pos] = ((Double) cVal).doubleValue();
- } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
- bigIntegers[pos] = (BigInteger) cVal;
- } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- bigDecimals[pos] = (BigDecimal) cVal;
- } else if (i.dataType() == DataTypes.DATE_TYPE) {
- dates[pos] = (Date) cVal;
- } else if (i.dataType() == DataTypes.STRING_TYPE) {
- strings[pos] = (String) cVal;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
- ints[pos] = ((EnumValue) cVal).ordinal;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- arrays[pos] = (ImmutableList) cVal;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
- maps[pos] = (ImmutableMap) cVal;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
- i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
- structs[pos] = (StructInstance) cVal;
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (cVal instanceof Id) {
- ids[pos] = (Id) cVal;
- } else {
- referenceables[pos] = (ReferenceableInstance) cVal;
- }
- } else {
- throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
- }
- }
-
- public Object get(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return null;
- }
-
- if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
- return bools[pos];
- } else if (i.dataType() == DataTypes.BYTE_TYPE) {
- return bytes[pos];
- } else if (i.dataType() == DataTypes.SHORT_TYPE) {
- return shorts[pos];
- } else if (i.dataType() == DataTypes.INT_TYPE) {
- return ints[pos];
- } else if (i.dataType() == DataTypes.LONG_TYPE) {
- return longs[pos];
- } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
- return floats[pos];
- } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
- return doubles[pos];
- } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
- return bigIntegers[pos];
- } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- return bigDecimals[pos];
- } else if (i.dataType() == DataTypes.DATE_TYPE) {
- return dates[pos];
- } else if (i.dataType() == DataTypes.STRING_TYPE) {
- return strings[pos];
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
- return ((EnumType) i.dataType()).fromOrdinal(ints[pos]);
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- return arrays[pos];
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
- return maps[pos];
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
- i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
- return structs[pos];
- } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (ids[pos] != null) {
- return ids[pos];
- } else {
- return referenceables[pos];
- }
- } else {
- throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
- }
- }
-
- public void setNull(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
- nullFlags[nullPos] = true;
- }
-
- /*
- * Use only for json serialization
- * @nonpublic
- */
- @Override
- public Map<String, Object> getValuesMap() throws MetadataException {
-
- Map<String,Object> m = new HashMap<>();
- for (String attr : fieldMapping.fields.keySet()) {
- m.put(attr, get(attr));
- }
- return m;
- }
-
- public boolean getBoolean(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BOOLEAN_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.BOOLEAN_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.BOOLEAN_TYPE.nullValue();
- }
-
- return bools[pos];
- }
-
- public byte getByte(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BYTE_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.BYTE_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.BYTE_TYPE.nullValue();
- }
-
- return bytes[pos];
- }
-
- public short getShort(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.SHORT_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.SHORT_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.SHORT_TYPE.nullValue();
- }
-
- return shorts[pos];
- }
-
- public int getInt(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
-
- if (i.dataType() != DataTypes.INT_TYPE && !(i.dataType() instanceof EnumType)) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.INT_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.INT_TYPE.nullValue();
- }
-
- return ints[pos];
- }
-
- public long getLong(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.LONG_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.LONG_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.LONG_TYPE.nullValue();
- }
-
- return longs[pos];
- }
-
- public float getFloat(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.FLOAT_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.FLOAT_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.FLOAT_TYPE.nullValue();
- }
-
- return floats[pos];
- }
-
- public double getDouble(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.DOUBLE_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.DOUBLE_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.DOUBLE_TYPE.nullValue();
- }
-
- return doubles[pos];
- }
-
- public BigInteger getBigInt(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BIGINTEGER_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.BIGINTEGER_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.BIGINTEGER_TYPE.nullValue();
- }
-
- return bigIntegers[pos];
- }
-
- public BigDecimal getBigDecimal(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BIGDECIMAL_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.BIGDECIMAL_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.BIGDECIMAL_TYPE.nullValue();
- }
-
- return bigDecimals[pos];
- }
-
- public Date getDate(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.DATE_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.DATE_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.DATE_TYPE.nullValue();
- }
-
- return dates[pos];
- }
-
- public String getString(String attrName) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.STRING_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic get method",
- attrName, getTypeName(), DataTypes.STRING_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- if (nullFlags[nullPos]) {
- return DataTypes.STRING_TYPE.nullValue();
- }
-
- return strings[pos];
- }
-
- public void setBoolean(String attrName, boolean val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BOOLEAN_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.BOOLEAN_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = false;
- bools[pos] = val;
- }
-
- public void setByte(String attrName, byte val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BYTE_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.BYTE_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = false;
- bytes[pos] = val;
- }
-
- public void setShort(String attrName, short val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.SHORT_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.SHORT_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = false;
- shorts[pos] = val;
- }
-
- public void setInt(String attrName, int val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.INT_TYPE && !(i.dataType() instanceof EnumType)) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.INT_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = false;
- ints[pos] = val;
- }
-
- public void setLong(String attrName, long val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.LONG_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.LONG_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = false;
- longs[pos] = val;
- }
-
- public void setFloat(String attrName, float val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.FLOAT_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.FLOAT_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = false;
- floats[pos] = val;
- }
-
- public void setDouble(String attrName, double val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.DOUBLE_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.DOUBLE_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = false;
- doubles[pos] = val;
- }
-
- public void setBigInt(String attrName, BigInteger val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BIGINTEGER_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.BIGINTEGER_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = val == null;
- bigIntegers[pos] = val;
- }
-
- public void setBigDecimal(String attrName, BigDecimal val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.BIGDECIMAL_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.BIGDECIMAL_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = val == null;
- bigDecimals[pos] = val;
- }
-
- public void setDate(String attrName, Date val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.DATE_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.DATE_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = val == null;
- dates[pos] = val;
- }
-
- public void setString(String attrName, String val) throws MetadataException {
- AttributeInfo i = fieldMapping.fields.get(attrName);
- if (i == null) {
- throw new MetadataException(
- String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
- }
-
- if (i.dataType() != DataTypes.STRING_TYPE) {
- throw new MetadataException(
- String.format("Field %s for Struct %s is not a %s, call generic set method",
- attrName, getTypeName(), DataTypes.STRING_TYPE.getName()));
- }
-
- int pos = fieldMapping.fieldPos.get(attrName);
- int nullPos = fieldMapping.fieldNullPos.get(attrName);
-
- nullFlags[nullPos] = val == null;
- strings[pos] = val;
- }
-
- public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
- TypeUtils.outputVal("{", buf, prefix);
- if (s == null) {
- TypeUtils.outputVal("<null>\n", buf, "");
- return;
- }
- TypeUtils.outputVal("\n", buf, "");
- String fieldPrefix = prefix + "\t";
- for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
- String attrName = e.getKey();
- AttributeInfo i = e.getValue();
- Object aVal = s.get(attrName);
- TypeUtils.outputVal(attrName + " : ", buf, fieldPrefix);
- i.dataType().output(aVal, buf, "");
- TypeUtils.outputVal("\n", buf, "");
- }
- TypeUtils.outputVal("\n}\n", buf, "");
- }
-
- @Override
- public String toString() {
- try {
- StringBuilder buf = new StringBuilder();
- String prefix = "";
-
- fieldMapping.output(this, buf, prefix);
- return buf.toString();
-
- } catch (MetadataException me) {
- throw new RuntimeException(me);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AbstractDataType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AbstractDataType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AbstractDataType.java
deleted file mode 100755
index c4cf88b..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AbstractDataType.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-abstract class AbstractDataType<T> implements IDataType<T> {
-
- protected T convertNull(Multiplicity m) throws MetadataException {
- if (!m.nullAllowed()) {
- throw new ValueConversionException.NullConversionException(m);
- }
- return null;
- }
-
- @Override
- public void output(T val, Appendable buf, String prefix) throws MetadataException {
- TypeUtils.outputVal(val == null ? "<null>" : val.toString(), buf, prefix);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeDefinition.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeDefinition.java
deleted file mode 100755
index 976fbf2..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeDefinition.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.ParamChecker;
-
-public final class AttributeDefinition {
-
- public final String name;
- public final String dataTypeName;
- public final Multiplicity multiplicity;
- public final boolean isComposite;
- public final boolean isUnique;
- public final boolean isIndexable;
-
- /**
- * If this is a reference attribute, then the name of the attribute on the Class
- * that this refers to.
- */
- public final String reverseAttributeName;
-
- public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity,
- boolean isComposite, String reverseAttributeName) {
- this(name, dataTypeName, multiplicity, isComposite, false, true, reverseAttributeName);
-
- }
-
- public AttributeDefinition(String name, String dataTypeName,
- Multiplicity multiplicity, boolean isComposite, boolean isUnique,
- boolean isIndexable, String reverseAttributeName) {
- this.name = ParamChecker.notEmpty(name, "Attribute name");
- this.dataTypeName = ParamChecker.notEmpty(dataTypeName, "Attribute type");
- this.multiplicity = multiplicity;
- this.isComposite = isComposite;
- this.isUnique = isUnique;
- this.isIndexable = isIndexable;
- this.reverseAttributeName = ParamChecker.notEmptyIfNotNull(reverseAttributeName, "Reverse attribute name");
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- AttributeDefinition that = (AttributeDefinition) o;
-
- if (isComposite != that.isComposite) return false;
- if (isUnique != that.isUnique) return false;
- if (isIndexable != that.isIndexable) return false;
- if (!dataTypeName.equals(that.dataTypeName)) return false;
- if (!multiplicity.equals(that.multiplicity)) return false;
- if (!name.equals(that.name)) return false;
- if (reverseAttributeName != null
- ? !reverseAttributeName.equals(that.reverseAttributeName)
- : that.reverseAttributeName != null)
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = name.hashCode();
- result = 31 * result + dataTypeName.hashCode();
- result = 31 * result + multiplicity.hashCode();
- result = 31 * result + (isComposite ? 1 : 0);
- result = 31 * result + (isUnique ? 1 : 0);
- result = 31 * result + (isIndexable ? 1 : 0);
- result = 31 * result + (reverseAttributeName != null ? reverseAttributeName.hashCode() : 0);
- return result;
- }
-
- @Override
- public String toString() {
- return name;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeInfo.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeInfo.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeInfo.java
deleted file mode 100755
index 0bacaa0..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/AttributeInfo.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.util.Map;
-
-public class AttributeInfo {
- public final String name;
- public final Multiplicity multiplicity;
- public final boolean isComposite;
- public final boolean isUnique;
- public final boolean isIndexable;
- /**
- * If this is a reference attribute, then the name of the attribute on the Class
- * that this refers to.
- */
- public final String reverseAttributeName;
- private IDataType dataType;
-
- AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws MetadataException {
- this.name = def.name;
- this.dataType = (tempTypes != null && tempTypes.containsKey(def.dataTypeName)) ?
- tempTypes.get(def.dataTypeName) : t.getDataType(IDataType.class, def.dataTypeName);
- this.multiplicity = def.multiplicity;
- this.isComposite = def.isComposite;
- this.isUnique = def.isUnique;
- this.isIndexable = def.isIndexable;
- this.reverseAttributeName = def.reverseAttributeName;
- }
-
- public IDataType dataType() {
- return dataType;
- }
-
- void setDataType(IDataType dT) {
- dataType = dT;
- }
-
- @Override
- public String toString() {
- return "AttributeInfo{" +
- "name='" + name + '\'' +
- ", dataType=" + dataType +
- ", multiplicity=" + multiplicity +
- ", isComposite=" + isComposite +
- ", isUnique=" + isUnique +
- ", isIndexable=" + isIndexable +
- ", reverseAttributeName='" + reverseAttributeName + '\'' +
- '}';
- }
-
- public String toJson() throws JSONException {
- JSONObject json = new JSONObject();
- json.put("name", name);
- json.put("multiplicity", multiplicity.toJson());
- json.put("isComposite", isComposite);
- json.put("isUnique", isUnique);
- json.put("isIndexable", isIndexable);
- json.put("dataType", dataType.getName());
- json.put("reverseAttributeName", reverseAttributeName);
- return json.toString();
- }
-
- public static AttributeDefinition fromJson(String jsonStr) throws JSONException {
- JSONObject json = new JSONObject(jsonStr);
- String reverseAttr = null;
- if (json.has("reverseAttributeName")) {
- reverseAttr = json.getString("reverseAttributeName");
- }
- return new AttributeDefinition(json.getString("name"), json.getString("dataType"),
- Multiplicity.fromJson(json.getString("multiplicity")), json.getBoolean("isComposite"),
- json.getBoolean("isUnique"), json.getBoolean("isIndexable"), reverseAttr);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ClassType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ClassType.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ClassType.java
deleted file mode 100755
index 4e8d738..0000000
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/ClassType.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.types;
-
-import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
-public class ClassType extends HierarchicalType<ClassType, IReferenceableInstance>
- implements IConstructableType<IReferenceableInstance, ITypedReferenceableInstance> {
-
- public static final String TRAIT_NAME_SEP = "::";
-
- public final Map<AttributeInfo, List<String>> infoToNameMap;
-
- /**
- * Used when creating a ClassType, to support recursive Structs.
- */
- ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, int numFields) {
- super(typeSystem, ClassType.class, name, superTypes, numFields);
- infoToNameMap = null;
- }
-
- ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes,
- AttributeInfo... fields)
- throws MetadataException {
- super(typeSystem, ClassType.class, name, superTypes, fields);
- infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
- }
-
- @Override
- public DataTypes.TypeCategory getTypeCategory() {
- return DataTypes.TypeCategory.CLASS;
- }
-
- public void validateId(Id id) throws MetadataException {
- if (id != null) {
- ClassType cType = typeSystem.getDataType(ClassType.class, id.className);
- if (isSubType(cType.getName())) {
- return;
- }
- throw new MetadataException(
- String.format("Id %s is not valid for class %s", id, getName()));
- }
- }
-
- protected Id getId(Object val) throws MetadataException {
- if (val instanceof Referenceable) {
- return ((Referenceable) val).getId();
- }
- throw new MetadataException(String.format("Cannot get id from class %s", val.getClass()));
- }
-
- @Override
- public ITypedReferenceableInstance convert(Object val, Multiplicity m)
- throws MetadataException {
-
- if (val != null) {
- if (val instanceof ITypedReferenceableInstance) {
- ITypedReferenceableInstance tr = (ITypedReferenceableInstance) val;
- if (!tr.getTypeName().equals(getName())) {
- /*
- * If val is a subType instance; invoke convert on it.
- */
- ClassType valType = typeSystem.getDataType(superTypeClass, tr.getTypeName());
- if (valType.superTypePaths.containsKey(name)) {
- return valType.convert(val, m);
- }
- throw new ValueConversionException(this, val);
- }
- return tr;
- } else if (val instanceof Struct) {
- Struct s = (Struct) val;
- Referenceable r = null;
- Id id = null;
-
- if (!s.typeName.equals(getName())) {
- /*
- * If val is a subType instance; invoke convert on it.
- */
- ClassType valType = typeSystem.getDataType(superTypeClass, s.typeName);
- if (valType.superTypePaths.containsKey(name)) {
- return valType.convert(s, m);
- }
- throw new ValueConversionException(this, val);
- }
-
- if (val instanceof Referenceable) {
- r = (Referenceable) val;
- id = r.getId();
- }
-
- ITypedReferenceableInstance tr = r != null ?
- createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0]))
- : createInstance(id);
-
- if (id != null && id.isAssigned()) {
- return tr;
- }
-
- for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
- String attrKey = e.getKey();
- AttributeInfo i = e.getValue();
- Object aVal = s.get(attrKey);
- if (aVal != null &&
- i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (!i.isComposite) {
- aVal = ((IReferenceableInstance) aVal).getId();
- }
- }
-
- try {
- tr.set(attrKey, aVal);
- } catch (ValueConversionException ve) {
- throw new ValueConversionException(this, val, ve);
- }
- }
-
- return tr;
- } else if (val instanceof ReferenceableInstance) {
- validateId(((ReferenceableInstance) val).getId());
- return (ReferenceableInstance) val;
- } else {
- throw new ValueConversionException(this, val);
- }
- }
- if (!m.nullAllowed()) {
- throw new ValueConversionException.NullConversionException(m);
- }
- return null;
- }
-
- @Override
- public ITypedReferenceableInstance createInstance() throws MetadataException {
- return createInstance((String[])null);
- }
-
- public ITypedReferenceableInstance createInstance(String... traitNames)
- throws MetadataException {
- return createInstance(null, traitNames);
- }
-
- public ITypedReferenceableInstance createInstance(Id id, String... traitNames)
- throws MetadataException {
- return createInstanceWithTraits(id, null, traitNames);
- }
-
- public ITypedReferenceableInstance createInstanceWithTraits(Id id, Referenceable r,
- String... traitNames)
- throws MetadataException {
-
- ImmutableMap.Builder<String, ITypedStruct> b
- = new ImmutableBiMap.Builder<String, ITypedStruct>();
- if (traitNames != null) {
- for (String t : traitNames) {
- TraitType tType = typeSystem.getDataType(TraitType.class, t);
- IStruct iTraitObject = r == null ? null : r.getTrait(t);
- ITypedStruct trait = iTraitObject == null ? tType.createInstance() :
- tType.convert(iTraitObject, Multiplicity.REQUIRED);
- b.put(t, trait);
- }
- }
-
- return new ReferenceableInstance(id == null ? new Id(getName()) : id,
- getName(),
- fieldMapping,
- new boolean[fieldMapping.fields.size()],
- fieldMapping.numBools == 0 ? null : new boolean[fieldMapping.numBools],
- fieldMapping.numBytes == 0 ? null : new byte[fieldMapping.numBytes],
- fieldMapping.numShorts == 0 ? null : new short[fieldMapping.numShorts],
- fieldMapping.numInts == 0 ? null : new int[fieldMapping.numInts],
- fieldMapping.numLongs == 0 ? null : new long[fieldMapping.numLongs],
- fieldMapping.numFloats == 0 ? null : new float[fieldMapping.numFloats],
- fieldMapping.numDoubles == 0 ? null : new double[fieldMapping.numDoubles],
- fieldMapping.numBigDecimals == 0 ? null
- : new BigDecimal[fieldMapping.numBigDecimals],
- fieldMapping.numBigInts == 0 ? null : new BigInteger[fieldMapping.numBigInts],
- fieldMapping.numDates == 0 ? null : new Date[fieldMapping.numDates],
- fieldMapping.numStrings == 0 ? null : new String[fieldMapping.numStrings],
- fieldMapping.numArrays == 0 ? null : new ImmutableList[fieldMapping.numArrays],
- fieldMapping.numMaps == 0 ? null : new ImmutableMap[fieldMapping.numMaps],
- fieldMapping.numStructs == 0 ? null : new StructInstance[fieldMapping.numStructs],
- fieldMapping.numReferenceables == 0 ? null
- : new ReferenceableInstance[fieldMapping.numReferenceables],
- fieldMapping.numReferenceables == 0 ? null : new Id[fieldMapping.numReferenceables],
- b.build());
- }
-
- @Override
- public void output(IReferenceableInstance s, Appendable buf, String prefix)
- throws MetadataException {
- fieldMapping.output(s, buf, prefix);
- }
-
- @Override
- public List<String> getNames(AttributeInfo info) {
- return infoToNameMap.get(info);
- }
-}
\ No newline at end of file
[55/58] [abbrv] incubator-atlas git commit: show arrow pointing
inwards for input tab
Posted by ve...@apache.org.
show arrow pointing inwards for input tab
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/2fa32eca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/2fa32eca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/2fa32eca
Branch: refs/heads/master
Commit: 2fa32eca1774e51264266ea0143a291624e2bc06
Parents: 0e18270
Author: Mouly Kumar <mk...@solarcity.com>
Authored: Mon Jun 15 21:50:23 2015 -0700
Committer: Mouly Kumar <mk...@solarcity.com>
Committed: Mon Jun 15 21:50:23 2015 -0700
----------------------------------------------------------------------
.../public/modules/lineage/lineageController.js | 21 +++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/2fa32eca/dashboard/v2/public/modules/lineage/lineageController.js
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/modules/lineage/lineageController.js b/dashboard/v2/public/modules/lineage/lineageController.js
index daecccb..368e9a5 100755
--- a/dashboard/v2/public/modules/lineage/lineageController.js
+++ b/dashboard/v2/public/modules/lineage/lineageController.js
@@ -136,6 +136,20 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
'translate(' + margin.left + ',' + margin.right + ')');
//arrow
svg.append("svg:defs").append("svg:marker").attr("id", "arrow").attr("viewBox", "0 0 10 10").attr("refX", 26).attr("refY", 5).attr("markerUnits", "strokeWidth").attr("markerWidth", 6).attr("markerHeight", 9).attr("orient", "auto").append("svg:path").attr("d", "M 0 0 L 10 5 L 0 10 z");
+
+ //marker for input type graph
+ svg.append("svg:defs")
+ .append("svg:marker")
+ .attr("id", "input-arrow")
+ .attr("viewBox", "0 0 10 10")
+ .attr("refX", -15)
+ .attr("refY", 5)
+ .attr("markerUnits", "strokeWidth")
+ .attr("markerWidth", 6)
+ .attr("markerHeight", 9)
+ .attr("orient", "auto")
+ .append("svg:path")
+ .attr("d", "M -2 5 L 8 0 L 8 10 z");
var root = data;
@@ -199,7 +213,12 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
//.style('stroke', function(d) { return d.target.level; })
.style('stroke', 'green')
.attr('d', diagonal);
- link.attr("marker-end", "url(#arrow)");
+
+ if($scope.type === 'inputs') {
+ link.attr("marker-start", "url(#input-arrow)");//if input
+ } else {
+ link.attr("marker-end", "url(#arrow)");//if input
+ }
}
[31/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
deleted file mode 100755
index ac9e2fa..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanVertex;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Graph;
-import com.tinkerpop.blueprints.GraphQuery;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.Set;
-import java.util.UUID;
-
-/**
- * Utility class for graph operations.
- */
-public final class GraphHelper {
-
- private static final Logger LOG = LoggerFactory.getLogger(GraphHelper.class);
-
- private GraphHelper() {
- }
-
- public static Vertex createVertexWithIdentity(Graph graph,
- ITypedReferenceableInstance typedInstance,
- Set<String> superTypeNames) {
- final Vertex vertexWithIdentity = createVertexWithoutIdentity(
- graph, typedInstance.getTypeName(), typedInstance.getId(), superTypeNames);
-
- // add identity
- final String guid = UUID.randomUUID().toString();
- vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid);
-
- return vertexWithIdentity;
- }
-
- public static Vertex createVertexWithoutIdentity(Graph graph,
- String typeName,
- Id typedInstanceId,
- Set<String> superTypeNames) {
- final Vertex vertexWithoutIdentity = graph.addVertex(null);
-
- // add type information
- vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
-
- // add super types
- for (String superTypeName : superTypeNames) {
- ((TitanVertex) vertexWithoutIdentity).addProperty(
- Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
- }
-
- // add version information
- vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
-
- // add timestamp information
- vertexWithoutIdentity.setProperty(
- Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
-
- return vertexWithoutIdentity;
- }
-
- public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,
- String edgeLabel) {
- LOG.debug("Adding edge for {} -> label {} -> {}",
- fromVertex, edgeLabel, toVertex);
-
- return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
- }
-
- public static Vertex findVertexByGUID(TitanGraph titanGraph,
- String value) {
- LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
-
- GraphQuery query = titanGraph.query()
- .has(Constants.GUID_PROPERTY_KEY, value);
- Iterator<Vertex> results = query.vertices().iterator();
- // returning one since guid should be unique
- return results.hasNext() ? results.next() : null;
- }
-
- public static String vertexString(final Vertex vertex) {
- StringBuilder properties = new StringBuilder();
- for (String propertyKey : vertex.getPropertyKeys()) {
- properties.append(propertyKey)
- .append("=").append(vertex.getProperty(propertyKey))
- .append(", ");
- }
-
- return "v[" + vertex.getId() + "], Properties[" + properties + "]";
- }
-
- public static String edgeString(final Edge edge) {
- return "e[" + edge.getLabel() + "], ["
- + edge.getVertex(Direction.OUT)
- + " -> " + edge.getLabel() + " -> "
- + edge.getVertex(Direction.IN)
- + "]";
- }
-
-/*
- public static void dumpToLog(final Graph graph) {
- LOG.debug("*******************Graph Dump****************************");
- LOG.debug("Vertices of {}", graph);
- for (Vertex vertex : graph.getVertices()) {
- LOG.debug(vertexString(vertex));
- }
-
- LOG.debug("Edges of {}", graph);
- for (Edge edge : graph.getEdges()) {
- LOG.debug(edgeString(edge));
- }
- LOG.debug("*******************Graph Dump****************************");
- }
-*/
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java
deleted file mode 100755
index c2879c4..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.google.inject.throwingproviders.CheckedProvider;
-import com.tinkerpop.blueprints.Graph;
-
-public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
-
- @Override
- T get();
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java
deleted file mode 100755
index af81edc..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.google.inject.Provides;
-import com.thinkaurelius.titan.core.TitanFactory;
-import com.thinkaurelius.titan.core.TitanGraph;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Singleton;
-import java.util.Iterator;
-
-/**
- * Default implementation for Graph Provider that doles out Titan Graph.
- */
-public class TitanGraphProvider implements GraphProvider<TitanGraph> {
-
- private static final Logger LOG = LoggerFactory.getLogger(TitanGraphProvider.class);
-
- /**
- * Constant for the configuration property that indicates the prefix.
- */
- private static final String METADATA_PREFIX = "metadata.graph.";
-
- private static Configuration getConfiguration() throws MetadataException {
- PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties();
-
- Configuration graphConfig = new PropertiesConfiguration();
-
- final Iterator<String> iterator = configProperties.getKeys();
- while (iterator.hasNext()) {
- String key = iterator.next();
- if (key.startsWith(METADATA_PREFIX)) {
- String value = (String) configProperties.getProperty(key);
- key = key.substring(METADATA_PREFIX.length());
- graphConfig.setProperty(key, value);
- LOG.info("Using graph property {}={}", key, value);
- }
- }
-
- return graphConfig;
- }
-
- @Override
- @Singleton
- @Provides
- public TitanGraph get() {
- Configuration config;
- try {
- config = getConfiguration();
- } catch (MetadataException e) {
- throw new RuntimeException(e);
- }
-
- return TitanFactory.open(config);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java
deleted file mode 100755
index 5b6ec34..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java
+++ /dev/null
@@ -1,655 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
-import it.unimi.dsi.fastutil.bytes.ByteArrayList;
-import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
-import it.unimi.dsi.fastutil.floats.FloatArrayList;
-import it.unimi.dsi.fastutil.ints.IntArrayList;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import it.unimi.dsi.fastutil.shorts.ShortArrayList;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.IConstructableType;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class AttributeStores {
-
- private static final Object NULL_VAL = new Object();
-
- static IAttributeStore createStore(AttributeInfo i) throws RepositoryException {
- switch (i.dataType().getTypeCategory()) {
- case PRIMITIVE:
- if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
- return new BooleanAttributeStore(i);
- } else if (i.dataType() == DataTypes.BYTE_TYPE) {
- return new ByteAttributeStore(i);
- } else if (i.dataType() == DataTypes.SHORT_TYPE) {
- new ShortAttributeStore(i);
- } else if (i.dataType() == DataTypes.INT_TYPE) {
- return new IntAttributeStore(i);
- } else if (i.dataType() == DataTypes.LONG_TYPE) {
- return new LongAttributeStore(i);
- } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
- return new FloatAttributeStore(i);
- } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
- return new DoubleAttributeStore(i);
- } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
- return new BigIntStore(i);
- } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- return new BigDecimalStore(i);
- } else if (i.dataType() == DataTypes.DATE_TYPE) {
- return new DateStore(i);
- } else if (i.dataType() == DataTypes.STRING_TYPE) {
- return new StringStore(i);
- } else if (i.dataType() == DataTypes.STRING_TYPE) {
- return new StringStore(i);
- } else {
- throw new RepositoryException(
- String.format("Unknown datatype %s", i.dataType()));
- }
- case ENUM:
- return new IntAttributeStore(i);
- case ARRAY:
- return new ImmutableListStore(i);
- case MAP:
- return new ImmutableMapStore(i);
- case STRUCT:
- return new StructStore(i);
- case CLASS:
- return new IdStore(i);
- default:
- throw new RepositoryException(
- String.format("Unknown Category for datatype %s", i.dataType()));
- }
- }
-
- static abstract class AbstractAttributeStore implements IAttributeStore {
- final BooleanArrayList nullList;
- final Map<Integer, Map<String, Object>> hiddenVals;
- AttributeInfo attrInfo;
-
- AbstractAttributeStore(AttributeInfo attrInfo) {
- this.attrInfo = attrInfo;
- this.nullList = new BooleanArrayList();
- hiddenVals = new HashMap<Integer, Map<String, Object>>();
- }
-
- final void setNull(int pos, boolean flag) {
- nullList.set(pos, flag);
- }
-
- final boolean getNull(int pos) {
- return nullList.get(pos);
- }
-
- void storeHiddenVals(int pos, IConstructableType type, StructInstance instance)
- throws RepositoryException {
- List<String> attrNames = type.getNames(attrInfo);
- Map<String, Object> m = hiddenVals.get(pos);
- if (m == null) {
- m = new HashMap<String, Object>();
- hiddenVals.put(pos, m);
- }
- for (int i = 2; i < attrNames.size(); i++) {
- String attrName = attrNames.get(i);
- int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
- int colPos = instance.fieldMapping().fieldPos.get(attrName);
- if (instance.nullFlags[nullPos]) {
- m.put(attrName, NULL_VAL);
- } else {
- //m.put(attrName, instance.bools[colPos]);
- store(instance, colPos, attrName, m);
- }
- }
- }
-
- void loadHiddenVals(int pos, IConstructableType type, StructInstance instance)
- throws RepositoryException {
- List<String> attrNames = type.getNames(attrInfo);
- Map<String, Object> m = hiddenVals.get(pos);
- for (int i = 2; i < attrNames.size(); i++) {
- String attrName = attrNames.get(i);
- int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
- int colPos = instance.fieldMapping().fieldPos.get(attrName);
- Object val = m == null ? NULL_VAL : m.get(attrName);
- if (val == NULL_VAL) {
- instance.nullFlags[nullPos] = true;
- } else {
- instance.nullFlags[nullPos] = false;
- load(instance, colPos, val);
- }
- }
- }
-
- @Override
- public void store(int pos, IConstructableType type, StructInstance instance)
- throws RepositoryException {
- List<String> attrNames = type.getNames(attrInfo);
- String attrName = attrNames.get(0);
- int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
- int colPos = instance.fieldMapping().fieldPos.get(attrName);
- nullList.set(pos, instance.nullFlags[nullPos]);
-
- if (pos == nullList.size()) {
- nullList.add(instance.nullFlags[nullPos]);
- } else {
- nullList.set(pos, instance.nullFlags[nullPos]);
- }
- //list.set(pos, instance.bools[colPos]);
- store(instance, colPos, pos);
-
- if (attrNames.size() > 1) {
- storeHiddenVals(pos, type, instance);
- }
- }
-
- @Override
- public void load(int pos, IConstructableType type, StructInstance instance)
- throws RepositoryException {
- List<String> attrNames = type.getNames(attrInfo);
- String attrName = attrNames.get(0);
- int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
- int colPos = instance.fieldMapping().fieldPos.get(attrName);
-
- if (nullList.get(pos)) {
- instance.nullFlags[nullPos] = true;
- } else {
- instance.nullFlags[nullPos] = false;
- load(instance, colPos, pos);
- }
-
- if (attrNames.size() > 1) {
- loadHiddenVals(pos, type, instance);
- }
- }
-
- /*
- * store the value from colPos in instance into the list.
- */
- protected abstract void store(StructInstance instance, int colPos, int pos)
- throws RepositoryException;
-
- /*
- * load the value from pos in list into colPos in instance.
- */
- protected abstract void load(StructInstance instance, int colPos, int pos)
- throws RepositoryException;
-
- /*
- * store the value from colPos in map as attrName
- */
- protected abstract void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m);
-
- /*
- * load the val into colPos in instance.
- */
- protected abstract void load(StructInstance instance, int colPos, Object val);
-
- }
-
- static abstract class PrimitiveAttributeStore extends AbstractAttributeStore
- implements IAttributeStore {
-
-
- public PrimitiveAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- }
-
- }
-
- static class BooleanAttributeStore extends PrimitiveAttributeStore {
-
- final BooleanArrayList list;
-
- BooleanAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = new BooleanArrayList();
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.bools[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.bools[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.bools[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.bools[colPos] = (Boolean) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- list.size(pos + 1);
- nullList.size(pos + 1);
- }
- }
-
- static class ByteAttributeStore extends PrimitiveAttributeStore {
-
- final ByteArrayList list;
-
- ByteAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = new ByteArrayList();
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.bytes[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.bytes[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.bytes[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.bytes[colPos] = (Byte) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- list.size(pos + 1);
- nullList.size(pos + 1);
- }
- }
-
- static class ShortAttributeStore extends PrimitiveAttributeStore {
-
- final ShortArrayList list;
-
- ShortAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = new ShortArrayList();
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.shorts[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.shorts[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.shorts[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.shorts[colPos] = (Short) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- list.size(pos + 1);
- nullList.size(pos + 1);
- }
- }
-
- static class IntAttributeStore extends PrimitiveAttributeStore {
-
- final IntArrayList list;
-
- IntAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = new IntArrayList();
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.ints[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.ints[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.ints[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.ints[colPos] = (Integer) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- list.size(pos + 1);
- nullList.size(pos + 1);
- }
- }
-
- static class LongAttributeStore extends PrimitiveAttributeStore {
-
- final LongArrayList list;
-
- LongAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = new LongArrayList();
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.longs[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.longs[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.longs[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.longs[colPos] = (Long) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- list.size(pos + 1);
- nullList.size(pos + 1);
- }
- }
-
- static class FloatAttributeStore extends PrimitiveAttributeStore {
-
- final FloatArrayList list;
-
- FloatAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = new FloatArrayList();
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.floats[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.floats[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.floats[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.floats[colPos] = (Float) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- list.size(pos + 1);
- nullList.size(pos + 1);
- }
- }
-
- static class DoubleAttributeStore extends PrimitiveAttributeStore {
-
- final DoubleArrayList list;
-
- DoubleAttributeStore(AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = new DoubleArrayList();
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.doubles[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.doubles[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.doubles[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.doubles[colPos] = (Double) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- list.size(pos + 1);
- nullList.size(pos + 1);
- }
- }
-
- static abstract class ObjectAttributeStore<T> extends AbstractAttributeStore {
-
- final ArrayList<T> list;
-
- ObjectAttributeStore(Class<T> cls, AttributeInfo attrInfo) {
- super(attrInfo);
- this.list = Lists.newArrayList((T) null);
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- while (list.size() < pos + 1) {
- list.add((T) null);
- }
- nullList.size(pos + 1);
- }
- }
-
- static class BigIntStore extends ObjectAttributeStore<BigInteger> {
-
- public BigIntStore(AttributeInfo attrInfo) {
- super(BigInteger.class, attrInfo);
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.bigIntegers[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.bigIntegers[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.bigIntegers[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.bigIntegers[colPos] = (BigInteger) val;
- }
-
- }
-
- static class BigDecimalStore extends ObjectAttributeStore<BigDecimal> {
-
- public BigDecimalStore(AttributeInfo attrInfo) {
- super(BigDecimal.class, attrInfo);
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.bigDecimals[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.bigDecimals[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.bigDecimals[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.bigDecimals[colPos] = (BigDecimal) val;
- }
-
- }
-
- static class DateStore extends ObjectAttributeStore<Date> {
-
- public DateStore(AttributeInfo attrInfo) {
- super(Date.class, attrInfo);
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.dates[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.dates[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.dates[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.dates[colPos] = (Date) val;
- }
-
- }
-
- static class StringStore extends ObjectAttributeStore<String> {
-
- public StringStore(AttributeInfo attrInfo) {
- super(String.class, attrInfo);
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.strings[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.strings[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.strings[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.strings[colPos] = (String) val;
- }
-
- }
-
- static class IdStore extends ObjectAttributeStore<Id> {
-
- public IdStore(AttributeInfo attrInfo) {
- super(Id.class, attrInfo);
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.ids[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.ids[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.ids[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.ids[colPos] = (Id) val;
- }
-
- }
-
- static class ImmutableListStore extends ObjectAttributeStore<ImmutableList> {
-
- public ImmutableListStore(AttributeInfo attrInfo) {
- super(ImmutableList.class, attrInfo);
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.arrays[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.arrays[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.arrays[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.arrays[colPos] = (ImmutableList) val;
- }
-
- }
-
- static class ImmutableMapStore extends ObjectAttributeStore<ImmutableMap> {
-
- public ImmutableMapStore(AttributeInfo attrInfo) {
- super(ImmutableMap.class, attrInfo);
- }
-
- protected void store(StructInstance instance, int colPos, int pos) {
- list.set(pos, instance.maps[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, int pos) {
- instance.maps[colPos] = list.get(pos);
- }
-
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.maps[colPos]);
- }
-
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.maps[colPos] = (ImmutableMap) val;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java
deleted file mode 100755
index 465aa12..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-
-import java.util.ArrayList;
-
-public class ClassStore extends HierarchicalTypeStore {
-
- final ArrayList<ImmutableList<String>> traitNamesStore;
- final ClassType classType;
-
- public ClassStore(MemRepository repository, ClassType hierarchicalType)
- throws RepositoryException {
- super(repository, hierarchicalType);
- classType = hierarchicalType;
- traitNamesStore = new ArrayList<ImmutableList<String>>();
- }
-
- void store(ReferenceableInstance i) throws RepositoryException {
- super.store(i);
- int pos = idPosMap.get(i.getId());
- traitNamesStore.set(pos, i.getTraits());
- }
-
- public void ensureCapacity(int pos) throws RepositoryException {
- super.ensureCapacity(pos);
- while (traitNamesStore.size() < pos + 1) {
- traitNamesStore.add(null);
- }
- }
-
- boolean validate(MemRepository repo, Id id) throws RepositoryException {
- if (id.isUnassigned()) {
- throw new RepositoryException(String.format("Invalid Id (unassigned) : %s", id));
- }
- Integer pos = idPosMap.get(id);
- if (pos == null) {
- throw new RepositoryException(String.format("Invalid Id (unknown) : %s", id));
- }
-
- String typeName = typeNameList.get(pos);
- if (typeName != hierarchicalType.getName()) {
- throw new RepositoryException(
- String.format("Invalid Id (incorrect typeName, type is %s) : %s",
- typeName, id));
- }
-
- return true;
- }
-
- /*
- * - assumes id is already validated
- */
- ReferenceableInstance createInstance(MemRepository repo, Id id) throws RepositoryException {
- Integer pos = idPosMap.get(id);
- String typeName = typeNameList.get(pos);
- if (typeName != hierarchicalType.getName()) {
- return repo.getClassStore(typeName).createInstance(repo, id);
- }
-
- ImmutableList<String> traitNames = traitNamesStore.get(pos);
- String[] tNs = traitNames.toArray(new String[]{});
-
- try {
- ReferenceableInstance r = (ReferenceableInstance) classType.createInstance(id, tNs);
- return r;
- } catch (MetadataException me) {
- throw new RepositoryException(me);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java
deleted file mode 100755
index 118f7fa..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
-import org.apache.hadoop.metadata.typesystem.types.IConstructableType;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-public abstract class HierarchicalTypeStore {
-
- final MemRepository repository;
- final IConstructableType hierarchicalType;
- final ArrayList<String> typeNameList;
- final ImmutableMap<AttributeInfo, IAttributeStore> attrStores;
- final ImmutableList<HierarchicalTypeStore> superTypeStores;
-
-
- /**
- * Map Id to position in storage lists.
- */
- Map<Id, Integer> idPosMap;
-
- List<Integer> freePositions;
-
- int nextPos;
-
- /**
- * Lock for each Class/Trait.
- */
- ReentrantReadWriteLock lock;
-
- HierarchicalTypeStore(MemRepository repository, HierarchicalType hierarchicalType)
- throws RepositoryException {
- this.hierarchicalType = (IConstructableType) hierarchicalType;
- this.repository = repository;
- ImmutableMap.Builder<AttributeInfo, IAttributeStore> b
- = new ImmutableBiMap.Builder<AttributeInfo,
- IAttributeStore>();
- typeNameList = Lists.newArrayList((String) null);
- ImmutableList<AttributeInfo> l = hierarchicalType.immediateAttrs;
- for (AttributeInfo i : l) {
- b.put(i, AttributeStores.createStore(i));
- }
- attrStores = b.build();
-
- ImmutableList.Builder<HierarchicalTypeStore> b1
- = new ImmutableList.Builder<HierarchicalTypeStore>();
- Set<String> allSuperTypeNames = hierarchicalType.getAllSuperTypeNames();
- for (String s : allSuperTypeNames) {
- b1.add(repository.getStore(s));
- }
- superTypeStores = b1.build();
-
- nextPos = 0;
- idPosMap = new HashMap<Id, Integer>();
- freePositions = new ArrayList<Integer>();
-
- lock = new ReentrantReadWriteLock();
- }
-
- /**
- * Assign a storage position to an Id.
- * - try to assign from freePositions
- * - ensure storage capacity.
- * - add entry in idPosMap.
- * @param id
- * @return
- * @throws RepositoryException
- */
- int assignPosition(Id id) throws RepositoryException {
-
- int pos = -1;
- if (!freePositions.isEmpty()) {
- pos = freePositions.remove(0);
- } else {
- pos = nextPos++;
- ensureCapacity(pos);
- }
-
- idPosMap.put(id, pos);
-
- for (HierarchicalTypeStore s : superTypeStores) {
- s.assignPosition(id);
- }
-
- return pos;
- }
-
- /**
- * - remove from idPosMap
- * - add to freePositions.
- * @throws RepositoryException
- */
- void releaseId(Id id) {
-
- Integer pos = idPosMap.get(id);
- if (pos != null) {
- idPosMap.remove(id);
- freePositions.add(pos);
-
- for (HierarchicalTypeStore s : superTypeStores) {
- s.releaseId(id);
- }
- }
- }
-
- void acquireReadLock() {
- lock.readLock().lock();
- }
-
- void acquireWriteLock() {
- lock.writeLock().lock();
- }
-
- void releaseReadLock() {
- lock.readLock().unlock();
- }
-
- void releaseWriteLock() {
- lock.writeLock().unlock();
- }
-
- protected void storeFields(int pos, StructInstance s) throws RepositoryException {
- for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
- IAttributeStore attributeStore = e.getValue();
- attributeStore.store(pos, hierarchicalType, s);
- }
- }
-
- protected void loadFields(int pos, StructInstance s) throws RepositoryException {
- for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
- IAttributeStore attributeStore = e.getValue();
- attributeStore.load(pos, hierarchicalType, s);
- }
- }
-
- /**
- * - store the typeName
- * - store the immediate attributes in the respective IAttributeStore
- * - call store on each SuperType.
- * @param i
- * @throws RepositoryException
- */
- void store(ReferenceableInstance i) throws RepositoryException {
- int pos = idPosMap.get(i.getId());
- typeNameList.set(pos, i.getTypeName());
- storeFields(pos, i);
-
- for (HierarchicalTypeStore s : superTypeStores) {
- s.store(i);
- }
- }
-
- /**
- * - copy over the immediate attribute values from the respective IAttributeStore
- * - call load on each SuperType.
- * @param i
- * @throws RepositoryException
- */
- void load(ReferenceableInstance i) throws RepositoryException {
- int pos = idPosMap.get(i.getId());
- loadFields(pos, i);
-
- for (HierarchicalTypeStore s : superTypeStores) {
- s.load(i);
- }
- }
-
- public void ensureCapacity(int pos) throws RepositoryException {
- while (typeNameList.size() < pos + 1) {
- typeNameList.add(null);
- }
- for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
- IAttributeStore attributeStore = e.getValue();
- attributeStore.ensureCapacity(pos);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java
deleted file mode 100755
index 9a3f05c..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
-import org.apache.hadoop.metadata.typesystem.types.IConstructableType;
-
-public interface IAttributeStore {
- /**
- * Store the attribute's value from the 'instance' into this store.
- * @param pos
- * @param instance
- * @throws RepositoryException
- */
- void store(int pos, IConstructableType type, StructInstance instance)
- throws RepositoryException;
-
- /**
- * load the Instance with the value from position 'pos' for the attribute.
- * @param pos
- * @param instance
- * @throws RepositoryException
- */
- void load(int pos, IConstructableType type, StructInstance instance) throws RepositoryException;
-
- /**
- * Ensure store have space for the given pos.
- * @param pos
- * @throws RepositoryException
- */
- void ensureCapacity(int pos) throws RepositoryException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java
deleted file mode 100755
index 74deccb..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java
+++ /dev/null
@@ -1,329 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.DiscoverInstances;
-import org.apache.hadoop.metadata.repository.IRepository;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.persistence.MapIds;
-import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicInteger;
-
-public class MemRepository implements IRepository {
-
-/*
- public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
- public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd");
-*/
-
- final TypeSystem typeSystem;
- /*
- * A Store for each Class and Trait.
- */
- final Map<String, HierarchicalTypeStore> typeStores;
- final AtomicInteger ID_SEQ = new AtomicInteger(0);
-
- public MemRepository(TypeSystem typeSystem) {
- this.typeSystem = typeSystem;
- this.typeStores = new HashMap<>();
- }
-
-/*
- @Override
- public DateFormat getDateFormat() {
- return dateFormat;
- }
-
- @Override
- public DateFormat getTimestampFormat() {
- return timestampFormat;
- }
-
- @Override
- public boolean allowNullsInCollections() {
- return false;
- }
-*/
-
- @Override
- public Id newId(String typeName) {
- return new Id("" + ID_SEQ.incrementAndGet(), 0, typeName);
- }
-
- /**
- * 1. traverse the Object Graph from i and create idToNewIdMap : Map[Id, Id],
- * also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance]
- * - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType
- * - traverse Structs
- * - traverse Traits.
- * 1b. Ensure that every newId has an associated Instance.
- * 2. Traverse oldIdToInstance map create newInstances : List[ITypedReferenceableInstance]
- * - create a ITypedReferenceableInstance.
- * replace any old References ( ids or object references) with new Ids.
- * 3. Traverse over newInstances
- * - ask ClassStore to assign a position to the Id.
- * - for Instances with Traits, assign a position for each Trait
- * - invoke store on the nwInstance.
- *
- * Recovery:
- * - on each newInstance, invoke releaseId and delete on its ClassStore and Traits' Stores.
- *
- * @param i
- * @return
- * @throws org.apache.hadoop.metadata.repository.RepositoryException
- */
- public ITypedReferenceableInstance create(IReferenceableInstance i) throws RepositoryException {
-
- DiscoverInstances discoverInstances = new DiscoverInstances(this);
-
- /*
- * Step 1: traverse the Object Graph from i and create idToNewIdMap : Map[Id, Id],
- * also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance]
- * - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType
- * - traverse Structs
- * - traverse Traits.
- */
- try {
- new ObjectGraphWalker(typeSystem, discoverInstances, i).walk();
- } catch (MetadataException me) {
- throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
- }
-
- /*
- * Step 1b: Ensure that every newId has an associated Instance.
- */
- for (Id oldId : discoverInstances.idToNewIdMap.keySet()) {
- if (!discoverInstances.idToInstanceMap.containsKey(oldId)) {
- throw new RepositoryException(String.format("Invalid Object Graph: " +
- "Encountered an unassignedId %s that is not associated with an Instance",
- oldId));
- }
- }
-
- /* Step 2: Traverse oldIdToInstance map create newInstances :
- List[ITypedReferenceableInstance]
- * - create a ITypedReferenceableInstance.
- * replace any old References ( ids or object references) with new Ids.
- */
- List<ITypedReferenceableInstance> newInstances
- = new ArrayList<ITypedReferenceableInstance>();
- ITypedReferenceableInstance retInstance = null;
- Set<ClassType> classTypes = new TreeSet<ClassType>();
- Set<TraitType> traitTypes = new TreeSet<TraitType>();
- for (IReferenceableInstance transientInstance : discoverInstances.idToInstanceMap
- .values()) {
- try {
- ClassType cT = typeSystem
- .getDataType(ClassType.class, transientInstance.getTypeName());
- ITypedReferenceableInstance newInstance = cT
- .convert(transientInstance, Multiplicity.REQUIRED);
- newInstances.add(newInstance);
-
- classTypes.add(cT);
- for (String traitName : newInstance.getTraits()) {
- TraitType tT = typeSystem.getDataType(TraitType.class, traitName);
- traitTypes.add(tT);
- }
-
- if (newInstance.getId() == i.getId()) {
- retInstance = newInstance;
- }
-
- /*
- * Now replace old references with new Ids
- */
- MapIds mapIds = new MapIds(discoverInstances.idToNewIdMap);
- new ObjectGraphWalker(typeSystem, mapIds, newInstances).walk();
-
- } catch (MetadataException me) {
- throw new RepositoryException(
- String.format("Failed to create Instance(id = %s",
- transientInstance.getId()), me);
- }
- }
-
- /*
- * 3. Acquire Class and Trait Storage locks.
- * - acquire them in a stable order (super before subclass, classes before traits
- */
- for (ClassType cT : classTypes) {
- HierarchicalTypeStore st = typeStores.get(cT.getName());
- st.acquireWriteLock();
- }
-
- for (TraitType tT : traitTypes) {
- HierarchicalTypeStore st = typeStores.get(tT.getName());
- st.acquireWriteLock();
- }
-
-
- /*
- * 4. Traverse over newInstances
- * - ask ClassStore to assign a position to the Id.
- * - for Instances with Traits, assign a position for each Trait
- * - invoke store on the nwInstance.
- */
- try {
- for (ITypedReferenceableInstance instance : newInstances) {
- HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
- st.assignPosition(instance.getId());
- for (String traitName : instance.getTraits()) {
- HierarchicalTypeStore tt = typeStores.get(traitName);
- tt.assignPosition(instance.getId());
- }
- }
-
- for (ITypedReferenceableInstance instance : newInstances) {
- HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
- st.store((ReferenceableInstance) instance);
- for (String traitName : instance.getTraits()) {
- HierarchicalTypeStore tt = typeStores.get(traitName);
- tt.store((ReferenceableInstance) instance);
- }
- }
- } catch (RepositoryException re) {
- for (ITypedReferenceableInstance instance : newInstances) {
- HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
- st.releaseId(instance.getId());
- }
- throw re;
- } finally {
- for (ClassType cT : classTypes) {
- HierarchicalTypeStore st = typeStores.get(cT.getName());
- st.releaseWriteLock();
- }
-
- for (TraitType tT : traitTypes) {
- HierarchicalTypeStore st = typeStores.get(tT.getName());
- st.releaseWriteLock();
- }
- }
-
- return retInstance;
- }
-
- public ITypedReferenceableInstance update(ITypedReferenceableInstance i)
- throws RepositoryException {
- throw new RepositoryException("not implemented");
- }
-
- public void delete(ITypedReferenceableInstance i) throws RepositoryException {
- throw new RepositoryException("not implemented");
- }
-
- public ITypedReferenceableInstance get(Id id) throws RepositoryException {
-
- try {
- ReplaceIdWithInstance replacer = new ReplaceIdWithInstance(this);
- ObjectGraphWalker walker = new ObjectGraphWalker(typeSystem, replacer);
- replacer.setWalker(walker);
- ITypedReferenceableInstance r = getDuringWalk(id, walker);
- walker.walk();
- return r;
- } catch (MetadataException me) {
- throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
- }
- }
-
- /*
- * - Id must be valid; Class must be valid.
- * - Ask ClassStore to createInstance.
- * - Ask ClassStore to load instance.
- * - load instance traits
- * - add to GraphWalker
- */
- ITypedReferenceableInstance getDuringWalk(Id id, ObjectGraphWalker walker)
- throws RepositoryException {
- ClassStore cS = getClassStore(id.getTypeName());
- if (cS == null) {
- throw new RepositoryException(String.format("Unknown Class %s", id.getTypeName()));
- }
- cS.validate(this, id);
- ReferenceableInstance r = cS.createInstance(this, id);
- cS.load(r);
- for (String traitName : r.getTraits()) {
- HierarchicalTypeStore tt = typeStores.get(traitName);
- tt.load(r);
- }
-
- walker.addRoot(r);
- return r;
- }
-
- HierarchicalTypeStore getStore(String typeName) {
- return typeStores.get(typeName);
- }
-
- ClassStore getClassStore(String typeName) {
- return (ClassStore) getStore(typeName);
- }
-
- public void defineClass(ClassType type) throws RepositoryException {
- HierarchicalTypeStore s = new ClassStore(this, type);
- typeStores.put(type.getName(), s);
- }
-
- public void defineTrait(TraitType type) throws RepositoryException {
- HierarchicalTypeStore s = new TraitStore(this, type);
- typeStores.put(type.getName(), s);
- }
-
- public void defineTypes(List<HierarchicalType> types) throws RepositoryException {
- List<TraitType> tTypes = new ArrayList<TraitType>();
- List<ClassType> cTypes = new ArrayList<ClassType>();
-
- for (HierarchicalType h : types) {
- if (h.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
- tTypes.add((TraitType) h);
- } else {
- cTypes.add((ClassType) h);
- }
- }
-
- Collections.sort(tTypes);
- Collections.sort(cTypes);
-
- for (TraitType tT : tTypes) {
- defineTrait(tT);
- }
-
- for (ClassType cT : cTypes) {
- defineClass(cT);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java
deleted file mode 100755
index f366d24..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-public class ReplaceIdWithInstance implements ObjectGraphWalker.NodeProcessor {
-
- public final Map<Id, ITypedReferenceableInstance> idToInstanceMap;
- final MemRepository repository;
- ObjectGraphWalker walker;
-
- public ReplaceIdWithInstance(MemRepository repository) {
- this.repository = repository;
- idToInstanceMap = new HashMap<>();
- }
-
- void setWalker(ObjectGraphWalker walker) {
- this.walker = walker;
- }
-
- @Override
- public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
- if (nd.attributeName == null) {
- // do nothing
- } else if (!nd.aInfo.isComposite || nd.value == null) {
- // do nothing
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
-
- if (nd.value != null && nd.value instanceof Id) {
- Id id = (Id) nd.value;
- ITypedReferenceableInstance r = getInstance(id);
- nd.instance.set(nd.attributeName, r);
- }
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
- DataTypes.ArrayType aT = (DataTypes.ArrayType) nd.aInfo.dataType();
- nd.instance.set(nd.attributeName,
- convertToInstances((ImmutableCollection) nd.value, nd.aInfo.multiplicity, aT));
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
- DataTypes.MapType mT = (DataTypes.MapType) nd.aInfo.dataType();
- nd.instance.set(nd.attributeName,
- convertToInstances((ImmutableMap) nd.value, nd.aInfo.multiplicity, mT));
- }
- }
-
- ImmutableCollection<?> convertToInstances(ImmutableCollection<?> val,
- Multiplicity m, DataTypes.ArrayType arrType)
- throws MetadataException {
-
- if (val == null ||
- arrType.getElemType().getTypeCategory() != DataTypes.TypeCategory.CLASS) {
- return val;
- }
-
- ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder()
- : ImmutableList.builder();
- Iterator it = val.iterator();
- while (it.hasNext()) {
- Object elem = it.next();
- if (elem instanceof Id) {
- Id id = (Id) elem;
- elem = getInstance(id);
- }
-
- b.add(elem);
-
- }
- return b.build();
- }
-
- ImmutableMap<?, ?> convertToInstances(ImmutableMap val, Multiplicity m,
- DataTypes.MapType mapType)
- throws MetadataException {
-
- if (val == null ||
- (mapType.getKeyType().getTypeCategory() != DataTypes.TypeCategory.CLASS &&
- mapType.getValueType().getTypeCategory() != DataTypes.TypeCategory.CLASS)) {
- return val;
- }
- ImmutableMap.Builder b = ImmutableMap.builder();
- Iterator<Map.Entry> it = val.entrySet().iterator();
- while (it.hasNext()) {
- Map.Entry elem = it.next();
- Object oldKey = elem.getKey();
- Object oldValue = elem.getValue();
- Object newKey = oldKey;
- Object newValue = oldValue;
-
- if (oldKey instanceof Id) {
- Id id = (Id) elem;
- ITypedReferenceableInstance r = getInstance(id);
- }
-
- if (oldValue instanceof Id) {
- Id id = (Id) elem;
- ITypedReferenceableInstance r = getInstance(id);
- }
-
- b.put(newKey, newValue);
- }
- return b.build();
- }
-
- ITypedReferenceableInstance getInstance(Id id) throws MetadataException {
-
- ITypedReferenceableInstance r = idToInstanceMap.get(id);
- if (r == null) {
- r = repository.get(id);
- idToInstanceMap.put(id, r);
- walker.addRoot(r);
- }
- return r;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java
deleted file mode 100755
index 502c691..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-
-import java.util.Collection;
-import java.util.Map;
-
-public class StructStore extends AttributeStores.AbstractAttributeStore implements IAttributeStore {
-
- final StructType structType;
- final ImmutableMap<AttributeInfo, IAttributeStore> attrStores;
-
- StructStore(AttributeInfo aInfo) throws RepositoryException {
- super(aInfo);
- this.structType = (StructType) aInfo.dataType();
- ImmutableMap.Builder<AttributeInfo, IAttributeStore> b = new ImmutableBiMap.Builder<>();
- Collection<AttributeInfo> l = structType.fieldMapping.fields.values();
- for (AttributeInfo i : l) {
- b.put(i, AttributeStores.createStore(i));
- }
- attrStores = b.build();
-
- }
-
- @Override
- protected void store(StructInstance instance, int colPos, int pos) throws RepositoryException {
- StructInstance s = instance.structs[colPos];
- for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
- IAttributeStore attributeStore = e.getValue();
- attributeStore.store(pos, structType, s);
- }
- }
-
- @Override
- protected void load(StructInstance instance, int colPos, int pos) throws RepositoryException {
- StructInstance s = (StructInstance) structType.createInstance();
- instance.structs[colPos] = s;
- for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
- IAttributeStore attributeStore = e.getValue();
- attributeStore.load(pos, structType, s);
- }
- }
-
- @Override
- protected void store(StructInstance instance, int colPos, String attrName,
- Map<String, Object> m) {
- m.put(attrName, instance.structs[colPos]);
- }
-
- @Override
- protected void load(StructInstance instance, int colPos, Object val) {
- instance.structs[colPos] = (StructInstance) val;
- }
-
- @Override
- public void ensureCapacity(int pos) throws RepositoryException {
- for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
- IAttributeStore attributeStore = e.getValue();
- attributeStore.ensureCapacity(pos);
- }
- nullList.size(pos + 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java
deleted file mode 100755
index 69a1894..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-
-import java.util.ArrayList;
-
-public class TraitStore extends HierarchicalTypeStore {
-
- final ArrayList<String> classNameStore;
-
- public TraitStore(MemRepository repository, TraitType hierarchicalType)
- throws RepositoryException {
- super(repository, hierarchicalType);
- classNameStore = new ArrayList<>();
- }
-
- void store(ReferenceableInstance i) throws RepositoryException {
- int pos = idPosMap.get(i.getId());
- StructInstance s = (StructInstance) i.getTrait(hierarchicalType.getName());
- super.storeFields(pos, s);
- classNameStore.set(pos, i.getTypeName());
- }
-
- void load(ReferenceableInstance i) throws RepositoryException {
- int pos = idPosMap.get(i.getId());
- StructInstance s = (StructInstance) i.getTrait(hierarchicalType.getName());
- super.loadFields(pos, s);
- }
-
- public void ensureCapacity(int pos) throws RepositoryException {
- super.ensureCapacity(pos);
- while (classNameStore.size() < pos + 1) {
- classNameStore.add(null);
- }
- }
-}
[49/58] [abbrv] incubator-atlas git commit: ATLAS-2 keystore and
certificate passwords need to be switched
Posted by ve...@apache.org.
ATLAS-2 keystore and certificate passwords need to be switched
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/0e18270b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/0e18270b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/0e18270b
Branch: refs/heads/master
Commit: 0e18270bdeb1ca32d6ec0a9211841d1a19feebc4
Parents: 2fbe8d9
Author: Jon Maron <jm...@hortonworks.com>
Authored: Sun Jun 14 13:23:35 2015 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Sun Jun 14 13:51:58 2015 -0400
----------------------------------------------------------------------
src/test/python/scripts/TestMetadata.py | 2 +-
.../java/org/apache/atlas/web/service/SecureEmbeddedServer.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0e18270b/src/test/python/scripts/TestMetadata.py
----------------------------------------------------------------------
diff --git a/src/test/python/scripts/TestMetadata.py b/src/test/python/scripts/TestMetadata.py
index b24b1c1..0efe96a 100644
--- a/src/test/python/scripts/TestMetadata.py
+++ b/src/test/python/scripts/TestMetadata.py
@@ -50,7 +50,7 @@ class TestMetadata(unittest.TestCase):
'org.apache.atlas.Main',
['-app', 'metadata_home/server/webapp/atlas'],
'metadata_home/conf:metadata_home/server/webapp/atlas/WEB-INF/classes:metadata_home/server/webapp/atlas/WEB-INF/lib\\*:metadata_home/libext\\*',
- ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-0Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
+ ['-Datlas.log.dir=metadata_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=metadata_home', '-Datlas.conf=metadata_home/conf', '-Xmx1024m'], 'metadata_home/logs')
else:
java_mock.assert_called_with(
'org.apache.atlas.Main',
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0e18270b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
index 63862a9..174dafb 100755
--- a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
+++ b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
@@ -59,11 +59,11 @@ public class SecureEmbeddedServer extends EmbeddedServer {
connector.setHost("0.0.0.0");
connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
- connector.setKeyPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
+ connector.setKeyPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
- connector.setPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
+ connector.setPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
return connector;
}
[51/58] [abbrv] incubator-atlas git commit: HDPDGI-69:Dileep:Use
details api to get LoadProcess name
Posted by ve...@apache.org.
HDPDGI-69:Dileep:Use details api to get LoadProcess name
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/1efb57bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/1efb57bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/1efb57bc
Branch: refs/heads/master
Commit: 1efb57bc453206b8377bb63e6f659d2861212697
Parents: d54a598
Author: Vishal Kadam <vi...@gmail.com>
Authored: Tue Jun 16 00:07:42 2015 -0400
Committer: Vishal Kadam <vi...@gmail.com>
Committed: Tue Jun 16 00:08:19 2015 -0400
----------------------------------------------------------------------
.../public/modules/lineage/lineageController.js | 66 +++++++++++++++++---
1 file changed, 57 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1efb57bc/dashboard/v2/public/modules/lineage/lineageController.js
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/modules/lineage/lineageController.js b/dashboard/v2/public/modules/lineage/lineageController.js
index daecccb..0227552 100755
--- a/dashboard/v2/public/modules/lineage/lineageController.js
+++ b/dashboard/v2/public/modules/lineage/lineageController.js
@@ -18,8 +18,9 @@
'use strict';
-angular.module('dgc.lineage').controller('LineageController', ['$element', '$scope', '$state', '$stateParams', 'lodash', 'LineageResource', 'd3',
- function($element, $scope, $state, $stateParams, _, LineageResource, d3) {
+angular.module('dgc.lineage').controller('LineageController', ['$element', '$scope', '$state', '$stateParams', 'lodash', 'LineageResource', 'd3', 'DetailsResource', '$q',
+ function($element, $scope, $state, $stateParams, _, LineageResource, d3, DetailsResource, $q) {
+ var guidsList = [];
function getLineageData(tableData, callRender) {
LineageResource.get({
@@ -27,15 +28,38 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
type: tableData.type
}, function lineageSuccess(response) {
if (!_.isEmpty(response.results.values.vertices)) {
- $scope.lineageData = transformData(response.results);
- if (callRender) {
- render();
- }
+ var allGuids = loadProcess(response.results.values.edges, response.results.values.vertices);
+ allGuids.then(function(res) {
+ guidsList = JSON.parse(res);
+ $scope.lineageData = transformData(response.results);
+ if (callRender) {
+ render();
+ }
+ });
}
$scope.requested = false;
});
}
+ function loadProcess(edges, vertices) {
+
+ var urlCalls = [];
+ var deferred = $q.defer();
+ for (var guid in edges) {
+ if (!vertices.hasOwnProperty(guid)) {
+ urlCalls.push(DetailsResource.get({
+ id: guid
+ }).$promise);
+ }
+
+ }
+ $q.all(urlCalls)
+ .then(function(results) {
+ deferred.resolve(JSON.stringify(results));
+ });
+ return deferred.promise;
+ }
+
$scope.type = $element.parent().attr('data-table-type');
$scope.requested = false;
@@ -67,10 +91,24 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
nodes = {};
function getNode(guid) {
+ var name, type;
+ if (vertices.hasOwnProperty(guid)) {
+ name = vertices[guid].values.name;
+ type = vertices[guid].values.vertexId.values.typeName;
+ } else {
+ var loadProcess = getLoadProcessTypes(guid);
+ if (typeof loadProcess !== "undefined") {
+ name = loadProcess.name;
+ type = loadProcess.typeName;
+ } else {
+ name = 'Load Process';
+ type = 'Load Process';
+ }
+ }
var vertex = {
guid: guid,
- name: vertices.hasOwnProperty(guid) ? vertices[guid].values.name : 'Load Process',
- type: vertices.hasOwnProperty(guid) ? vertices[guid].values.vertexId.values.typeName : 'LoadProcess'
+ name: name,
+ type: type
};
if (!nodes.hasOwnProperty(guid)) {
nodes[guid] = vertex;
@@ -78,6 +116,17 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
return nodes[guid];
}
+ function getLoadProcessTypes(guid) {
+ var procesRes = [];
+ angular.forEach(guidsList, function(value) {
+ if (value.id.id === guid) {
+ procesRes.name = value.values.name;
+ procesRes.typeName = value.typeName;
+ }
+ });
+ return procesRes;
+ }
+
function attachParent(edge, node) {
edge.forEach(function eachPoint(childGuid) {
var childNode = getNode(childGuid);
@@ -172,7 +221,6 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
.attr("y", "-18px")
.attr("width", "34px")
.attr("height", "34px");
-
nodeEnter.append('text')
.attr('x', function(d) {
return d.children || d._children ?
[20/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftTypesGen.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftTypesGen.scala b/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftTypesGen.scala
new file mode 100755
index 0000000..c020d46
--- /dev/null
+++ b/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftTypesGen.scala
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.thrift
+
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.MetadataException
+import org.apache.atlas.typesystem.TypesDef
+import org.apache.atlas.typesystem.types.{DataTypes, HierarchicalTypeDefinition, Multiplicity, TraitType, _}
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.io.Source
+import scala.util.{Failure, Success, Try}
+
+
+case class CompositeRelation(typeName: String, fieldName: String, reverseFieldName: Option[String])
+
+/**
+ * Convert a [[ThriftDef ThriftDef]] to
+ * [[TypesDef TypesDef]]. Currently there are several restrictions:
+ *
+ * - CppIncludes, SEnums are not allowed
+ * - The only include allowed is that of "share/fb303/if/fb303.thrift". This include is ignored.
+ * Any other include will trigger an exception
+ * - Namespaces, TypeDefs, Contants, Unions, Exceptions, and Service definitions are ignored.
+ * - So for fields typeDefs are not applied.
+ * - Field Constant values are ignored.
+ * - Type Annotations, XSD information is ignored.
+ *
+ * Thrift Structs can be mapped to Structs, Traits for Classes. The caller can specify their preference by
+ * providing the structNames, classNames and thriftNames parameters. A Struct that is not in one of these 3
+ * lists is not mapped.
+ *
+ * The ThriftDef doesn't specify if a relationship is composite. For e.g. in the thrift definition
+ * {{{
+ * struct Person {
+ 1: string name,
+ 2: Address addr,
+ }
+ struct Address {
+ 1: string street,
+ 2: string city,
+ }
+ * }}}
+ *
+ * If Person and Address are mapped to classes, you may not to make the Person -> Address a Composite relation.
+ * The caller can specify these in the 'compositeRelations' parameter.
+ *
+ */
+class ThriftTypesGen(val structNames: List[String], val classNames: List[String], val traitNames: List[String],
+ val compositeRelations: List[CompositeRelation]) {
+
+ private val LOG: Logger = LoggerFactory.getLogger(classOf[ThriftTypesGen])
+ private val FB_INCLUDE = "share/fb303/if/fb303.thrift"
+ /**
+ * for a (typeName, fieldName) specifies (isComposite, reverseName)
+ * if entry doesn't exist than field is not composite.
+ */
+ private var compositeRelsMap: Map[(String, String), (Boolean, Option[String])] = Map()
+
+ def apply(thriftResource: String): TypesDef = {
+ val tDef = parseThrift(thriftResource)
+
+ tDef.flatMap(buildCompositeRelations).flatMap(typesDef) match {
+ case Success(t) => t
+ case Failure(v) => throw v
+ }
+
+ }
+
+ def buildCompositeRelations(thriftDef: ThriftDef): Try[ThriftDef] = Try {
+
+ compositeRelations.foreach { cr =>
+
+ val sDef = thriftDef.structs.find(_.name == cr.typeName)
+ if (!sDef.isDefined) {
+ throw new MetadataException(s"Unknown Struct (${cr.typeName}) specified in CompositeRelation")
+
+ }
+ val fDef = sDef.get.fields.find(_.name == cr.fieldName)
+ if (!fDef.isDefined) {
+ throw new MetadataException(s"Unknown Field (${cr.fieldName}) specified in CompositeRelation")
+
+ }
+
+ compositeRelsMap = compositeRelsMap + ((cr.typeName, cr.fieldName) ->(true, cr.reverseFieldName))
+
+ if (cr.reverseFieldName.isDefined) {
+ val reverseStructName = dataTypeName(fDef.get.fieldType)
+ val reverseStructDef = thriftDef.structs.find(_.name == reverseStructName)
+ if (!reverseStructDef.isDefined) {
+ throw new MetadataException(s"Cannot find Struct $reverseStructName in CompositeRelation $cr")
+ }
+ val rfDef = reverseStructDef.get.fields.find(_.name == cr.reverseFieldName)
+ if (!rfDef.isDefined) {
+ throw new MetadataException(s"Unknown Reverse Field (${cr.reverseFieldName}) specified in CompositeRelation")
+ }
+
+ List(cr, CompositeRelation(reverseStructName, cr.reverseFieldName.get, Some(cr.fieldName)))
+
+ compositeRelsMap = compositeRelsMap +
+ ((reverseStructName, cr.reverseFieldName.get) ->(false, Some(cr.fieldName)))
+ }
+ }
+
+ thriftDef
+ }
+
+ def typesDef(thriftDef: ThriftDef): Try[TypesDef] = {
+ var tDef: Try[TypesDef] = Try {
+ TypesDef(Seq(), Seq(), Seq(), Seq())
+ }
+
+ tDef.flatMap((t: TypesDef) => includes(t, thriftDef.includes)).flatMap((t: TypesDef) => cppIncludes(t, thriftDef.cppIncludes))
+
+ tDef = tDef.flatMap((t: TypesDef) => includes(t, thriftDef.includes)).
+ flatMap((t: TypesDef) => cppIncludes(t, thriftDef.cppIncludes)).
+ flatMap((t: TypesDef) => namespaces(t, thriftDef.namespaces)).
+ flatMap((t: TypesDef) => constants(t, thriftDef.constants)).
+ flatMap((t: TypesDef) => senums(t, thriftDef.senums)).
+ flatMap((t: TypesDef) => enums(t, thriftDef.enums)).
+ flatMap((t: TypesDef) => structs(t, thriftDef.structs)).
+ flatMap((t: TypesDef) => unions(t, thriftDef.unions)).
+ flatMap((t: TypesDef) => exceptions(t, thriftDef.xceptions)).
+ flatMap((t: TypesDef) => services(t, thriftDef.services))
+
+
+ tDef
+ }
+
+ private def parseThrift(thriftResource: String): Try[ThriftDef] = {
+ Try {
+ LOG.debug("Parsing Thrift resource {}", thriftResource)
+ val is = getClass().getResourceAsStream(thriftResource)
+ val src: Source = Source.fromInputStream(is)
+ val thriftStr: String = src.getLines().mkString("\n")
+ val p = new ThriftParser
+ var thriftDef: Option[ThriftDef] = p(thriftStr)
+ thriftDef match {
+ case Some(s) => s
+ case None => {
+ LOG.debug("Parse for thrift resource {} failed", thriftResource)
+ throw new MetadataException(s"Failed to parse thrift resource: $thriftResource")
+ }
+ }
+ }
+ }
+
+ @throws[MetadataException]
+ private def dataTypeName(fT: FieldType): String = fT match {
+ case IdentifierType(n) => n
+ case BaseType(typ, _) => BASE_TYPES.toPrimitiveTypeName(typ)
+ case ListType(elemType, _, _) => DataTypes.arrayTypeName(dataTypeName(elemType))
+ case SetType(elemType, _, _) => DataTypes.arrayTypeName(dataTypeName(elemType))
+ case MapType(keyType, valueType, _, _) => DataTypes.mapTypeName(dataTypeName(keyType), dataTypeName(valueType))
+ }
+
+ private def enumValue(e: EnumValueDef, defId: Int): EnumValue = e match {
+ case EnumValueDef(value, Some(id), _) => new EnumValue(value, id.value)
+ case EnumValueDef(value, None, _) => new EnumValue(value, defId)
+ }
+
+ private def enumDef(td: TypesDef, e: EnumDef): Try[TypesDef] = {
+ Success(
+ td.copy(enumTypes = td.enumTypes :+
+ new EnumTypeDefinition(e.name, e.enumValues.zipWithIndex.map(t => enumValue(t._1, -t._2)): _*))
+ )
+ }
+
+ private def includeDef(td: TypesDef, i: IncludeDef): Try[TypesDef] = {
+ Try {
+ if (i.value != FB_INCLUDE) {
+ throw new MetadataException(s"Unsupported Include ${i.value}, only fb303.thrift is currently allowed.")
+ }
+ td
+ }
+ }
+
+ private def cppIncludeDef(td: TypesDef, i: CppIncludeDef): Try[TypesDef] = {
+ Try {
+ throw new MetadataException(s"Unsupported CppInclude ${i.value}.")
+ }
+ }
+
+ private def namespaceDef(td: TypesDef, i: NamespaceDef): Try[TypesDef] = {
+ Try {
+ LOG.debug(s"Ignoring Namespace definition $i")
+ td
+ }
+ }
+
+ private def constantDef(td: TypesDef, i: ConstDef): Try[TypesDef] = {
+ Try {
+ LOG.debug(s"Ignoring ConstantDef definition $i")
+ td
+ }
+ }
+
+ private def senumDef(td: TypesDef, i: SEnumDef): Try[TypesDef] = {
+ Try {
+ throw new MetadataException(s"Unsupported SEnums ${i}.")
+ }
+ }
+
+ private def fieldDef(typName: String, fd: FieldDef): AttributeDefinition = {
+ val name: String = fd.name
+ val dTName: String = dataTypeName(fd.fieldType)
+
+ var m: Multiplicity = Multiplicity.OPTIONAL
+
+ if (fd.requiredNess) {
+ m = Multiplicity.REQUIRED
+ }
+
+ fd.fieldType match {
+ case _: ListType => m = Multiplicity.COLLECTION
+ case _: SetType => m = Multiplicity.SET
+ case _ => ()
+ }
+
+ var isComposite = false
+ var reverseAttrName: String = null
+
+ val r = compositeRelsMap.get((typName, name))
+ if (r.isDefined) {
+ isComposite = r.get._1
+ if (r.get._2.isDefined) {
+ reverseAttrName = r.get._2.get
+ }
+ }
+ new AttributeDefinition(name, dTName, m, isComposite, reverseAttrName)
+ }
+
+ private def structDef(td: TypesDef, structDef: StructDef): Try[TypesDef] = Try {
+ val typeName: String = structDef.name
+
+ typeName match {
+ case t if structNames contains t => td.copy(structTypes = td.structTypes :+
+ new StructTypeDefinition(typeName, structDef.fields.map(fieldDef(typeName, _)).toArray))
+ case t: String if traitNames contains t => {
+ val ts = td.traitTypes :+
+ new HierarchicalTypeDefinition[TraitType](classOf[TraitType],
+ typeName, ImmutableList.of[String](), structDef.fields.map(fieldDef(typeName, _)).toArray)
+ td.copy(traitTypes = ts)
+ }
+ case t: String if classNames contains t => {
+ val cs = td.classTypes :+
+ new HierarchicalTypeDefinition[ClassType](classOf[ClassType],
+ typeName, ImmutableList.of[String](), structDef.fields.map(fieldDef(typeName, _)).toArray)
+ td.copy(classTypes = cs)
+ }
+ case _ => td
+ }
+ }
+
+ private def unionDef(td: TypesDef, i: UnionDef): Try[TypesDef] = {
+ Try {
+ LOG.debug(s"Ignoring Union definition $i")
+ td
+ }
+ }
+
+ private def exceptionDef(td: TypesDef, i: ExceptionDef): Try[TypesDef] = {
+ Try {
+ LOG.debug(s"Ignoring Exception definition $i")
+ td
+ }
+ }
+
+ private def serviceDef(td: TypesDef, i: ServiceDef): Try[TypesDef] = {
+ Try {
+ LOG.debug(s"Ignoring Service definition $i")
+ td
+ }
+ }
+
+ private def applyList[T](fn: (TypesDef, T) => Try[TypesDef])(td: TypesDef, l: List[T]): Try[TypesDef] = {
+ l.foldLeft[Try[TypesDef]](Success(td))((b, a) => b.flatMap({ Unit => fn(td, a)}))
+ }
+
+ private def includes = applyList(includeDef) _
+
+ private def cppIncludes = applyList(cppIncludeDef) _
+
+ private def namespaces = applyList(namespaceDef) _
+
+ private def constants = applyList(constantDef) _
+
+ private def enums = applyList(enumDef) _
+
+ private def senums = applyList(senumDef) _
+
+ private def structs = applyList(structDef) _
+
+ private def unions = applyList(unionDef) _
+
+ private def exceptions = applyList(exceptionDef) _
+
+ private def services = applyList(serviceDef) _
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/DynamicTypedStruct.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/DynamicTypedStruct.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/DynamicTypedStruct.scala
deleted file mode 100755
index 2670321..0000000
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/DynamicTypedStruct.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.dsl
-
-import org.apache.hadoop.metadata.typesystem.ITypedStruct
-import org.apache.hadoop.metadata.typesystem.types.{StructType, TypeSystem}
-
-import scala.language.dynamics
-
-class DynamicTypedStruct(val ts: ITypedStruct) extends Dynamic {
- def selectDynamic(name: String) = ts.get(name)
-
- def updateDynamic(name: String)(value: Any) {
- var value1 = value
- if (value != null && value.isInstanceOf[DynamicTypedStruct]) {
- value1 = value.asInstanceOf[DynamicTypedStruct].ts
- }
- ts.set(name, value1)
- }
-
- def dataType = TypeSystem.getInstance().getDataType(classOf[StructType], ts.getTypeName)
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala
deleted file mode 100755
index ba09342..0000000
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata
-
-import java.text.SimpleDateFormat
-
-import org.apache.hadoop.metadata.typesystem.json.{BigDecimalSerializer, BigIntegerSerializer, Serialization, TypedStructSerializer}
-import org.apache.hadoop.metadata.repository.memory.MemRepository
-import org.apache.hadoop.metadata.tools.dsl.DynamicTypedStruct
-import org.apache.hadoop.metadata.typesystem.persistence.StructInstance
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.typesystem.{IStruct, ITypedStruct}
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.{write => swrite}
-
-import scala.language.implicitConversions
-import scala.collection.JavaConversions._
-
-package object dsl {
-
- val defFormat = new DefaultFormats {
- override protected def dateFormatter = TypeSystem.getInstance().getDateFormat;
-
- override val typeHints = NoTypeHints
- }
-
- implicit val formats = defFormat + new TypedStructSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer
- val BOOLEAN_TYPE = DataTypes.BOOLEAN_TYPE
- val BYTE_TYPE = DataTypes.BYTE_TYPE
- val SHORT_TYPE = DataTypes.SHORT_TYPE
- val INT_TYPE = DataTypes.INT_TYPE
- val LONG_TYPE = DataTypes.LONG_TYPE
- val FLOAT_TYPE = DataTypes.FLOAT_TYPE
- val DOUBLE_TYPE = DataTypes.DOUBLE_TYPE
- val BIGINT_TYPE = DataTypes.BIGINTEGER_TYPE
- val BIGDECIMAL_TYPE = DataTypes.BIGDECIMAL_TYPE
- val DATE_TYPE = DataTypes.DATE_TYPE
- val STRING_TYPE = DataTypes.STRING_TYPE
- val ATTR_OPTIONAL = Multiplicity.OPTIONAL
- val ATTR_REQUIRED = Multiplicity.REQUIRED
-
- def repo = new MemRepository(ts)
-
- def arrayType(dT: IDataType[_]) = ts.defineArrayType(dT)
-
- def mapType(kT: IDataType[_], vT: IDataType[_]) = ts.defineMapType(kT, vT)
-
- def attrDef(name: String, dT: IDataType[_],
- m: Multiplicity = Multiplicity.OPTIONAL,
- isComposite: Boolean = false,
- reverseAttributeName: String = null) = {
- require(name != null)
- require(dT != null)
- new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName)
- }
-
- def listTypes = (ts.getTypeNames -- ts.getCoreTypes).sorted.toList.mkString("[", ",", "]")
-
- def ts = TypeSystem.getInstance
-
- def defineStructType(name: String, attrDef: AttributeDefinition*) = {
- require(name != null)
- ts.defineStructType(name, false, attrDef: _*)
- }
-
- def createInstance(typeName: String, jsonStr: String)(implicit formats: Formats) = {
- val j = parse(jsonStr)
- assert(j.isInstanceOf[JObject])
- var j1 = j.asInstanceOf[JObject]
- j1 = JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(typeName)) :: j1.obj)
- new DynamicTypedStruct(Extraction.extract[StructInstance](j1))
- }
-
- def createInstance(typeName: String) = {
- new DynamicTypedStruct(
- ts.getDataType(classOf[StructType], typeName).asInstanceOf[IConstructableType[IStruct, ITypedStruct]].createInstance())
- }
-
- implicit def dynTypedStructToTypedStruct(s: DynamicTypedStruct) = s.ts
-
- implicit def dynTypedStructToJson(s: DynamicTypedStruct)(implicit formats: Formats) = {
- Extraction.decompose(s.ts)(formats)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/Main.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/Main.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/Main.scala
deleted file mode 100755
index c95ca99..0000000
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/Main.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.simpleserver
-
-import akka.actor.{ActorSystem, Props}
-import akka.io.IO
-import com.typesafe.config.ConfigFactory
-import org.apache.hadoop.metadata.repository.memory.MemRepository
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import spray.can.Http
-
-/**
- * A Simple Spray based server to test the TypeSystem and MemRepository.
- *
- * @example {{{
- * -- Using the [[ https://github.com/jakubroztocil/httpie Httpie tool]]
- *
- * http GET localhost:9140/listTypeNames
- * pbpaste | http PUT localhost:9140/defineTypes
- * http GET localhost:9140/typeDetails typeNames:='["Department", "Person", "Manager"]'
- *
- * pbpaste | http PUT localhost:9140/createInstance
- * pbpaste | http GET localhost:9140/getInstance
- * }}}
- *
- * - On the Mac, pbpaste makes available what is copied to clipboard. Copy contents of resources/sampleTypes.json
- * - for createInstance resources/sampleInstance.json is an example
- * - for getInstance send an Id back, you can copy the output from createInstance.
- *
- */
-object Main extends App {
- val config = ConfigFactory.load()
- val host = config.getString("http.host")
- val port = config.getInt("http.port")
-
- implicit val system = ActorSystem("metadataservice")
-
- val typSys = TypeSystem.getInstance()
- val memRepo = new MemRepository(typSys)
-
- val api = system.actorOf(Props(new RestInterface(typSys, memRepo)), "httpInterface")
- IO(Http) ! Http.Bind(listener = api, interface = host, port = port)
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/MetadataActor.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/MetadataActor.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/MetadataActor.scala
deleted file mode 100755
index 6878411..0000000
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/MetadataActor.scala
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.simpleserver
-
-import akka.actor._
-import akka.util.Timeout
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.repository.memory.MemRepository
-import org.apache.hadoop.metadata.typesystem.json._
-import org.apache.hadoop.metadata.typesystem.persistence.Id
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.typesystem.{ITypedReferenceableInstance, TypesDef}
-import org.json4s.{Formats, NoTypeHints}
-import spray.httpx.Json4sSupport
-
-import scala.concurrent.duration._
-
-
-class MetadataActor(val typeSystem: TypeSystem, val memRepository: MemRepository) extends Actor with ActorLogging {
-
- import org.apache.hadoop.metadata.tools.simpleserver.MetadataProtocol._
-
- import scala.collection.JavaConversions._
- import scala.language.postfixOps
- implicit val timeout = Timeout(5 seconds)
-
-
- def receive = {
- case ListTypeNames() =>
- sender ! TypeNames(typeSystem.getTypeNames.toList)
-
- case GetTypeDetails(typeNames) =>
- val typesDef = TypesSerialization.convertToTypesDef(typeSystem, (d: IDataType[_]) => typeNames.contains(d.getName))
- sender ! TypeDetails(typesDef)
-
- case DefineTypes(typesDef: TypesDef) =>
- typesDef.enumTypes.foreach(typeSystem.defineEnumType(_))
-
- typeSystem.defineTypes(ImmutableList.copyOf(typesDef.structTypes.toArray),
- ImmutableList.copyOf(typesDef.traitTypes.toArray),
- ImmutableList.copyOf(typesDef.classTypes.toArray))
-
- var newTypes: List[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]] = Nil
- typesDef.traitTypes.foreach { tDef =>
- val nm = tDef.typeName
- newTypes = newTypes :+
- typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm)
- }
- typesDef.classTypes.foreach { tDef =>
- val nm = tDef.typeName
- newTypes = newTypes :+
- typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm)
- }
-
- memRepository.defineTypes(newTypes)
-
- sender ! TypesCreated
-
- case CreateInstance(i) =>
- val r = memRepository.create(i)
- sender ! InstanceCreated(r.getId)
-
- case GetInstance(id) =>
- val r = memRepository.get(id)
- sender ! InstanceDetails(r)
- }
-
-}
-
-object MetadataProtocol {
-
- case class ListTypeNames()
-
- case class TypeNames(typeNames: List[String])
-
- case class GetTypeDetails(typeNames: List[String])
-
- case class TypeDetails(types: TypesDef)
-
- case class DefineTypes(types: TypesDef)
-
- case class TypesCreated()
-
- case class CreateInstance(i: ITypedReferenceableInstance)
-
- case class InstanceCreated(id: Id)
-
- case class GetInstance(id: Id)
-
- case class InstanceDetails(i: ITypedReferenceableInstance)
-
-}
-
-
-trait Json4sProtocol extends Json4sSupport {
- val typeSystem: TypeSystem
- val memRepository: MemRepository
-
- implicit def json4sFormats: Formats =
- org.json4s.native.Serialization.formats(NoTypeHints) + new MultiplicitySerializer +
- new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer + new IdSerializer
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/RestInterface.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/RestInterface.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/RestInterface.scala
deleted file mode 100755
index b7f68fa..0000000
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/simpleserver/RestInterface.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.simpleserver
-
-import akka.actor._
-import akka.util.Timeout
-import org.apache.hadoop.metadata.repository.memory.MemRepository;
-import org.apache.hadoop.metadata.typesystem.{TypesDef, ITypedReferenceableInstance}
-import org.apache.hadoop.metadata.typesystem.persistence.Id
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import spray.http.StatusCodes
-import spray.routing._
-import scala.concurrent.duration._
-
-class Responder(val typeSystem: TypeSystem, val memRepository : MemRepository,
- requestContext:RequestContext, mdSvc:ActorRef) extends Actor with Json4sProtocol with ActorLogging {
- import org.apache.hadoop.metadata.tools.simpleserver.MetadataProtocol._
-
- def receive = {
-
- case typNames:TypeNames =>
- requestContext.complete(StatusCodes.OK, typNames)
- self ! PoisonPill
-
- case tD:TypeDetails =>
- requestContext.complete(StatusCodes.OK, tD)
- self ! PoisonPill
-
- case TypesCreated =>
- requestContext.complete(StatusCodes.OK)
- self ! PoisonPill
-
- case InstanceCreated(id) =>
- requestContext.complete(StatusCodes.OK, id)
-
- case InstanceDetails(i) =>
- requestContext.complete(StatusCodes.OK, i)
- }
-}
-
-class RestInterface(val typeSystem: TypeSystem, val memRepository : MemRepository) extends HttpServiceActor
-with RestApi {
- def receive = runRoute(routes)
-}
-
-
-trait RestApi extends HttpService with Json4sProtocol with ActorLogging { actor: Actor =>
- import MetadataProtocol._
- import scala.language.postfixOps
- import scala.concurrent.ExecutionContext.Implicits.global
-
- val typeSystem : TypeSystem
- val memRepository : MemRepository
-
- implicit val timeout = Timeout(10 seconds)
-
- import akka.pattern.{ask, pipe}
-
- val mdSvc = context.actorOf(Props(new MetadataActor(typeSystem, memRepository)))
-
- def routes: Route =
-
- path("listTypeNames") {
- get { requestContext =>
- val responder: ActorRef = createResponder(requestContext)
-
- pipe(mdSvc.ask(ListTypeNames))
-
- mdSvc.ask(ListTypeNames()).pipeTo(responder)
- }
- } ~
- path("typeDetails") {
- get {
- entity(as[GetTypeDetails]) { typeDetails => requestContext =>
- val responder = createResponder(requestContext)
- mdSvc.ask(typeDetails).pipeTo(responder)
- }
- }
- } ~
- path("defineTypes") {
- put {
- entity(as[TypesDef]) { typesDef => requestContext =>
- val responder = createResponder(requestContext)
- mdSvc.ask(DefineTypes(typesDef)).pipeTo(responder)
- }
- }
- } ~
- path("createInstance") {
- put {
- entity(as[ITypedReferenceableInstance]) { i => requestContext =>
- val responder = createResponder(requestContext)
- mdSvc.ask(CreateInstance(i)).pipeTo(responder)
- }
- }
- } ~
- path("getInstance") {
- get {
- entity(as[Id]) { id => requestContext =>
- val responder = createResponder(requestContext)
- mdSvc.ask(GetInstance(id)).pipeTo(responder)
- }
- }
- }
-
- def createResponder(requestContext:RequestContext) = {
- context.actorOf(Props(new Responder(typeSystem, memRepository, requestContext, mdSvc)))
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParser.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParser.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParser.scala
deleted file mode 100755
index fa0ea1e..0000000
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftParser.scala
+++ /dev/null
@@ -1,665 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.thrift
-
-import org.apache.hadoop.metadata.MetadataException
-import org.apache.hadoop.metadata.typesystem.types.DataTypes
-
-import scala.util.parsing.combinator.{ImplicitConversions, PackratParsers}
-import scala.util.parsing.combinator.lexical.StdLexical
-import scala.util.parsing.combinator.syntactical.StandardTokenParsers
-
-import scala.util.parsing.input.CharArrayReader._
-
-object BASE_TYPES extends Enumeration {
- val STRING = Value("string")
- val BINARY = Value("binary")
- val SLIST = Value("slist")
- val BOOLEAN = Value("bool")
- val BYTE = Value("byte")
- val I16 = Value("i16")
- val I32 = Value("i32")
- val I64 = Value("i64")
- val DOUBLE = Value("double")
-
- @throws[MetadataException]
- def toPrimitiveTypeName(t : BASE_TYPES.Value) : String = t match {
- case STRING => DataTypes.STRING_TYPE.getName
- case SLIST => DataTypes.STRING_TYPE.getName
- case BOOLEAN => DataTypes.BOOLEAN_TYPE.getName
- case BYTE => DataTypes.BYTE_TYPE.getName
- case I16 => DataTypes.SHORT_TYPE.getName
- case I32 => DataTypes.INT_TYPE.getName
- case I64 => DataTypes.LONG_TYPE.getName
- case DOUBLE => DataTypes.DOUBLE_TYPE.getName
- case _ => throw new MetadataException(s"Thrift BaseType ($t) not supported")
- }
-}
-
-object THRIFT_LANG extends Enumeration {
- val CPP = Value("cpp")
- val PHP = Value("php")
- val PY = Value("py")
- val PERL = Value("perl")
- val RUBY = Value("ruby")
- val SMLTK_CAT = Value("smalltalk.category")
- val SMLTK_PRE = Value("smalltalk.prefix")
- val JAVA = Value("java")
- val COCOA = Value("cocoa")
- val XSD = Value("xsd")
- val CSHARP = Value("csharp")
- val STAR = Value("*")
- val OTHER = Value("")
-}
-
-case class TypeAnnotation(name : String, value : String)
-case class CPPType(name : String)
-sealed trait FunctionType
-case class VoidType() extends FunctionType
-sealed trait FieldType extends FunctionType
-case class IdentifierType(name : String) extends FieldType
-case class BaseType(typ : BASE_TYPES.Value, typAnnotations :Option[List[TypeAnnotation]]) extends FieldType
-sealed trait ContainerType extends FieldType {
- def typAnnotations :Option[List[TypeAnnotation]]
-}
-case class MapType(keyType : FieldType, valueType : FieldType,
- cppType : Option[CPPType],
- typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType
-case class SetType(elemType : FieldType,
- cppType : Option[CPPType],
- typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType
-case class ListType(elemType : FieldType,
- cppType : Option[CPPType],
- typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType
-
-sealed trait ConstValue
-case class IntConstant(value : Int) extends ConstValue
-case class DoubleConstant(value : Double) extends ConstValue
-case class StringConstant(value : String) extends ConstValue
-case class IdConstant(value : String) extends ConstValue
-case class ConstantList(value : List[ConstValue]) extends ConstValue
-case class ConstantValuePair(first : ConstValue, second : ConstValue)
-case class ConstantMap(value : List[ConstantValuePair]) extends ConstValue
-
-case class ConstDef(fieldType : FieldType, id : String, value : ConstValue)
-
-case class TypeDef(name : String, fieldType : FieldType,
- typAnnotations :Option[List[TypeAnnotation]])
-case class EnumValueDef(value : String, id : Option[IntConstant], typAnnotations :Option[List[TypeAnnotation]])
-case class EnumDef(name : String, enumValues : List[EnumValueDef], typAnnotations :Option[List[TypeAnnotation]])
-
-case class SEnumDef(name : String, enumValues : List[String], typAnnotations :Option[List[TypeAnnotation]])
-
-case class FieldDef(id : Option[IntConstant], requiredNess : Boolean, fieldType : FieldType, name : String,
- fieldValue : Option[ConstValue], xsdOptional : Boolean, xsdNillable : Boolean,
- xsdAttributes: Option[XsdAttributes],
- typAnnotations :Option[List[TypeAnnotation]])
-
-case class XsdAttributes(fields : List[FieldDef])
-
-case class StructDef(name : String, xsdAll : Boolean, fields : List[FieldDef],
- typAnnotations :Option[List[TypeAnnotation]])
-
-case class UnionDef(val name : String, val xsdAll : Boolean,
- val fields : List[FieldDef],
- val typAnnotations :Option[List[TypeAnnotation]])
-
-case class ExceptionDef(val name : String,
- val fields : List[FieldDef],
- val typAnnotations :Option[List[TypeAnnotation]])
-
-case class FunctionDef(oneway : Boolean, returnType : FunctionType, name : String, parameters : List[FieldDef],
- throwFields : Option[List[FieldDef]], typAnnotations :Option[List[TypeAnnotation]])
-
-case class ServiceDef(name : String, superName : Option[String], functions : List[FunctionDef],
- typAnnotations :Option[List[TypeAnnotation]])
-
-case class IncludeDef(value : String)
-case class CppIncludeDef(val value : String)
-case class NamespaceDef(lang : THRIFT_LANG.Value, name : String, otherLang : Option[String] = None)
-
-case class ThriftDef(val includes : List[IncludeDef],
- val cppIncludes : List[CppIncludeDef],
- val namespaces : List[NamespaceDef],
- val constants : List[ConstDef],
- val typedefs : List[TypeDef],
- val enums : List[EnumDef],
- val senums : List[SEnumDef],
- val structs : List[StructDef],
- val unions : List[UnionDef],
- val xceptions : List[ExceptionDef],
- val services : List[ServiceDef]) {
-
- def this() = this(List(), List(), List(), List(), List(), List(), List(),
- List(), List(), List(), List())
-
- def this(a : IncludeDef) = this(a :: Nil, List(), List(), List(), List(), List(), List(),
- List(), List(), List(), List())
- def this(a : CppIncludeDef) = this(List(), a :: Nil, List(), List(), List(), List(), List(), List(),
- List(), List(), List())
- def this(a : NamespaceDef) = this(List(), List(), a :: Nil, List(), List(), List(), List(), List(), List(),
- List(), List())
- def this(a : ConstDef) = this(List(), List(), List(), a :: Nil, List(), List(), List(), List(), List(), List(),
- List())
- def this(a : TypeDef) = this(List(), List(), List(), List(), a :: Nil, List(), List(), List(), List(), List(), List())
- def this(a : EnumDef) = this(List(), List(), List(), List(), List(), a :: Nil, List(), List(),
- List(), List(), List())
- def this(a : SEnumDef) = this(List(), List(), List(), List(), List(), List(), a :: Nil, List(),
- List(), List(), List())
- def this(a : StructDef) = this(List(), List(), List(), List(), List(), List(), List(), a :: Nil,
- List(), List(), List())
- def this(a : UnionDef) = this(List(), List(), List(), List(), List(), List(), List(),
- List(), a :: Nil, List(), List())
- def this(a : ExceptionDef) = this(List(), List(), List(), List(), List(), List(), List(),
- List(), List(), a :: Nil, List())
- def this(a : ServiceDef) = this(List(), List(), List(), List(), List(), List(), List(),
- List(), List(), List(), a :: Nil)
-
-
- def plus(a : IncludeDef) = ThriftDef(includes.+:(a), cppIncludes, namespaces, constants, typedefs, enums, senums,
- structs, unions, xceptions, services)
- def plus(a : CppIncludeDef) = ThriftDef(includes, cppIncludes.+:(a), namespaces, constants, typedefs, enums, senums,
- structs, unions, xceptions, services)
- def plus(a : NamespaceDef) = ThriftDef(includes, cppIncludes, namespaces.+:(a), constants, typedefs, enums, senums,
- structs, unions, xceptions, services)
- def plus(a : ConstDef) = ThriftDef(includes, cppIncludes, namespaces, constants.+:(a), typedefs, enums, senums,
- structs, unions, xceptions, services)
- def plus(a : TypeDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs.+:(a), enums, senums,
- structs, unions, xceptions, services)
- def plus(a : EnumDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums.+:(a), senums,
- structs, unions, xceptions, services)
- def plus(a : SEnumDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums.+:(a),
- structs, unions, xceptions, services)
- def plus(a : StructDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
- structs.+:(a), unions, xceptions, services)
- def plus(a : UnionDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
- structs, unions.+:(a), xceptions, services)
- def plus(a : ExceptionDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
- structs, unions, xceptions.+:(a), services)
- def plus(a : ServiceDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
- structs, unions, xceptions, services.+:(a))
- def plus(a : ThriftDef) = ThriftDef(includes ::: a.includes,
- cppIncludes ::: a.cppIncludes,
- namespaces ::: a.namespaces,
- constants ::: a.constants,
- typedefs ::: a.typedefs,
- enums ::: a.enums,
- senums ::: a.senums,
- structs ::: a.structs,
- unions ::: a.unions,
- xceptions ::: a.xceptions,
- services ::: a.services)
-
-
-}
-
-trait ThriftKeywords {
- this : StandardTokenParsers =>
-
- import scala.language.implicitConversions
-
- protected case class Keyword(str: String)
-
- protected implicit def asParser(k: Keyword): Parser[String] = k.str
-
- protected val LPAREN = Keyword("(")
- protected val RPAREN = Keyword(")")
- protected val EQ = Keyword("=")
- protected val CPP_TYPE = Keyword("cpp_type")
- protected val LIST = Keyword("list")
- protected val LT = Keyword("<")
- protected val GT = Keyword(">")
- protected val SET = Keyword("set")
- protected val MAP = Keyword("map")
- protected val STRING = Keyword("string")
- protected val BINARY = Keyword("binary")
- protected val SLIST = Keyword("slist")
- protected val BOOL = Keyword("bool")
- protected val BYTE = Keyword("byte")
- protected val I16 = Keyword("i16")
- protected val I32 = Keyword("i32")
- protected val I64 = Keyword("i64")
- protected val DOUBLE = Keyword("double")
- protected val VOID = Keyword("void")
- protected val REQUIRED = Keyword("required")
- protected val OPTIONAL = Keyword("optional")
- protected val COLON = Keyword(":")
- protected val THROWS = Keyword("throws")
- protected val ONEWAY = Keyword("oneway")
- protected val EXTENDS = Keyword("extends")
- protected val SERVICE = Keyword("service")
- protected val EXCEPTION = Keyword("exception")
- protected val LBRACKET = Keyword("{")
- protected val RBRACKET = Keyword("}")
- protected val XSD_ATTRS = Keyword("xsd_attributes")
- protected val XSD_NILBLE = Keyword("xsd_nillable")
- protected val XSD_OPT = Keyword("xsd_optional")
- protected val XSD_ALL = Keyword("xsd_all")
- protected val UNION = Keyword("union")
- protected val LSQBRACKET = Keyword("[")
- protected val RSQBRACKET = Keyword("]")
- protected val CONST = Keyword("const")
- protected val STRUCT = Keyword("struct")
- protected val SENUM = Keyword("senum")
- protected val ENUM = Keyword("enum")
- protected val COMMA = Keyword(",")
- protected val SEMICOLON = Keyword(";")
- protected val TYPEDEF = Keyword("typedef")
- protected val INCLUDE = Keyword("include")
- protected val CPP_INCL = Keyword("cpp_include")
- protected val NAMESPACE = Keyword("namespace")
- protected val STAR = Keyword("*")
- protected val CPP_NS = Keyword("cpp_namespace")
- protected val PHP_NS = Keyword("php_namespace")
- protected val PY_NS = Keyword("py_module")
- protected val PERL_NS = Keyword("perl_package")
- protected val RUBY_NS = Keyword("ruby_namespace")
- protected val SMLTK_CAT = Keyword("smalltalk_category")
- protected val SMLTK_PRE = Keyword("smalltalk_prefix")
- protected val JAVA_NS = Keyword("java_package")
- protected val COCOA_NS = Keyword("cocoa_package")
- protected val XSD_NS = Keyword("xsd_namespace")
- protected val CSHARP_NS = Keyword("csharp_namespace")
-
- def isRequired(r : Option[String]) = r match {
- case Some(REQUIRED) => true
- case _ => false
- }
-
- def isXsdOptional(r : Option[String]) = r match {
- case Some(XSD_OPT) => true
- case _ => false
- }
-
- def isXsdNillable(r : Option[String]) = r match {
- case Some(XSD_NILBLE) => true
- case _ => false
- }
-
- def isXsdAll(r : Option[String]) = r match {
- case Some(XSD_ALL) => true
- case _ => false
- }
-
- def isOneWay(r : Option[String]) = r match {
- case Some(ONEWAY) => true
- case _ => false
- }
-
-}
-
-trait ThriftTypeRules extends ThriftKeywords {
- this : StandardTokenParsers =>
-
- def containterType : Parser[ContainerType] = mapType | setType | listType
-
- def setType = SET ~ cppType.? ~ LT ~ fieldType ~ GT ~ typeAnnotations.? ^^ {
- case s ~ ct ~ lt ~ t ~ gt ~ tA => SetType(t, ct, tA)
- }
-
- def listType = LIST ~ LT ~ fieldType ~ GT ~ cppType.? ~ typeAnnotations.? ^^ {
- case l ~ lt ~ t ~ gt ~ ct ~ tA => ListType(t, ct, tA)
- }
-
- def mapType = MAP ~ cppType.? ~ LT ~ fieldType ~ COMMA ~ fieldType ~ GT ~ typeAnnotations.? ^^ {
- case s ~ ct ~ lt ~ kt ~ c ~ vt ~ gt ~ tA => MapType(kt, vt, ct, tA)
- }
-
- def cppType : Parser[CPPType] = CPP_TYPE ~ stringLit ^^ { case c ~ s => CPPType(s)}
-
- def fieldType: Parser[FieldType] = ident ^^ {case i => IdentifierType(i)} |
- baseType |
- containterType
-
- def baseType : Parser[BaseType] = simpleBaseType ~ typeAnnotations.? ^^ { case s ~ t => BaseType(s, t)}
-
- def simpleBaseType : Parser[BASE_TYPES.Value] = STRING ^^^ BASE_TYPES.STRING |
- BINARY ^^^ BASE_TYPES.BINARY |
- SLIST ^^^ BASE_TYPES.SLIST |
- BOOL ^^^ BASE_TYPES.BOOLEAN |
- BYTE ^^^ BASE_TYPES.BYTE |
- I16 ^^^ BASE_TYPES.I16 |
- I32 ^^^ BASE_TYPES.I32 |
- I64 ^^^ BASE_TYPES.I64 |
- DOUBLE ^^^ BASE_TYPES.DOUBLE
-
- def typeAnnotations : Parser[List[TypeAnnotation]] =
- LPAREN ~ typeAnnotation.* ~ RPAREN ^^ { case l ~ t ~ r => t.toList}
-
- def typeAnnotation : Parser[TypeAnnotation] =
- (ident ~ EQ ~ stringLit ~ commaOrSemicolon.?) ^^ { case i ~ e ~ s ~ c => TypeAnnotation(i,s)}
-
- def commaOrSemicolon : Parser[String] = COMMA | SEMICOLON
-
-}
-
-/**
- * @todo extract Constant Rules into this Trait. This requires moving `hexConstant` here. But how to specify
- * type of `HexConstant`, it is a Path dependent Type tied to lexical member of ThriftParser.
- */
-trait ThriftConstantRules extends ThriftKeywords {
- this: StandardTokenParsers =>
-
- // def parseDouble(s: String) = try { Some(s.toDouble) } catch { case _ : Throwable => None }
- //
- // def constValue : Parser[ConstValue] = numericLit ^^ {
- // case n => parseDouble(n) match {
- // case Some(d) => DoubleConstant(d)
- // case _ => IntConstant(n.toInt)
- // }
- // } |
- // hexConstant ^^ { case h => IntConstant(Integer.parseInt(h, 16))} |
- // stringLit ^^ { case s => StringConstant(s)} |
- // ident ^^ { case i => IdConstant(i)} |
- // constList |
- // constMap
- //
- // def constValuePair = constValue ~ COLON ~ constValue ~ commaOrSemicolon.? ^^ {
- // case k ~ c ~ v ~ cs => ConstantValuePair(k,v)
- // }
- //
- // def constList = LSQBRACKET ~ (constValue <~ commaOrSemicolon).* ~ RSQBRACKET ^^ {
- // case l ~ vs ~ r => ConstantList(vs)
- // }
- //
- // def constMap = LBRACKET ~ constValuePair.* ~ RBRACKET ^^ {
- // case l ~ ps ~ r => ConstantMap(ps)
- // }
-}
-
-/**
- * A Parser for Thrift definition scripts.
- * Based on [[https://github.com/twitter/commons/blob/master/src/antlr/twitter/thrift/descriptors/AntlrThrift.g]].
- * Definition is parsed into a [[org.apache.hadoop.metadata.tools.thrift.ThriftDef ThriftDef]] structure.
- *
- * @example {{{
- * var p = new ThriftParser
- * var td : Option[ThriftDef] = p("""include "share/fb303/if/fb303.thrift"
- * namespace java org.apache.hadoop.hive.metastore.api
- * namespace php metastore
- * namespace cpp Apache.Hadoop.Hive
- * \""")
- * }}}
- *
- * @todo doesn't traverse includes directives. Includes are parsed into
- * [[org.apache.hadoop.metadata.tools.thrift.IncludeDef IncludeDef]] structures
- * but are not traversed.
- * @todo mixing in [[scala.util.parsing.combinator.PackratParsers PackratParsers]] is a placeholder. Need to
- * change specific grammar rules to `lazy val` and `Parser[Elem]` to `PackratParser[Elem]`. Will do based on
- * performance analysis.
- * @todo Error reporting
- */
-class ThriftParser extends StandardTokenParsers with ThriftKeywords with ThriftTypeRules with PackratParsers {
-
- import scala.language.higherKinds
-
- private val reservedWordsDelims : Seq[String] =
- this
- .getClass
- .getMethods
- .filter(_.getReturnType == classOf[Keyword])
- .map(_.invoke(this).asInstanceOf[Keyword].str)
-
- private val (thriftreservedWords : Seq[String], thriftdelims : Seq[String]) =
- reservedWordsDelims.partition(s => s.charAt(0).isLetter)
-
- override val lexical = new ThriftLexer(thriftreservedWords, thriftdelims)
-
- import lexical.HexConstant
- /** A parser which matches a hex constant */
- def hexConstant: Parser[String] =
- elem("string literal", _.isInstanceOf[HexConstant]) ^^ (_.chars)
-
- def apply(input: String): Option[ThriftDef] = {
- phrase(program)(new lexical.Scanner(input)) match {
- case Success(r, x) => Some(r)
- case Failure(m, x) => {
- None
- }
- case Error(m, x) => {
- None
- }
- }
- }
-
- def program = headers ~ definitions ^^ { case h ~ d => h plus d}
-
- def headers = header.* ^^ { case l => l.foldRight(new ThriftDef)((a,t) => t plus a)}
-
- def header = INCLUDE ~> stringLit ^^ { case s => new ThriftDef(IncludeDef(s))} |
- CPP_INCL ~> stringLit ^^ { case s => new ThriftDef(CppIncludeDef(s))} |
- NAMESPACE ~ ident ~ ident ^^ { case ns ~ t ~ n => new ThriftDef(NamespaceDef(THRIFT_LANG.OTHER, n, Some(t)))} |
- NAMESPACE ~ STAR ~ ident ^^ { case ns ~ s ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.STAR, i))} |
- CPP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.CPP, i))} |
- PHP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PHP, i))} |
- PY_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PY, i))} |
- PERL_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PERL, i))} |
- RUBY_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.RUBY, i))} |
- SMLTK_CAT ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.SMLTK_CAT, i))} |
- SMLTK_PRE ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.SMLTK_PRE, i))} |
- JAVA_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.JAVA, i))} |
- COCOA_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.COCOA, i))} |
- XSD_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.XSD, i))} |
- CSHARP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.CSHARP, i))}
-
- def definitions : Parser[ThriftDef] = definition.* ^^ {
- case l => l.foldRight(new ThriftDef)((a,t) => t plus a)
- }
-
- def definition : Parser[ThriftDef] = const ^^ { case c => new ThriftDef(c)} |
- typeDefinition |
- service ^^ { case s => new ThriftDef(s)}
-
-
- def typeDefinition : Parser[ThriftDef] = (typedef ^^ {case t => new ThriftDef(t)} |
- enum ^^ {case e => new ThriftDef(e)} |
- senum ^^ {case e => new ThriftDef(e)} |
- struct ^^ {case e => new ThriftDef(e)} |
- union ^^ {case e => new ThriftDef(e)} |
- xception ^^ {case e => new ThriftDef(e)}
- )
-
- def typedef : Parser[TypeDef] = TYPEDEF ~ fieldType ~ ident ~ typeAnnotations.? ^^ {
- case t ~ f ~ i ~ tA => TypeDef(i, f, tA)
- }
-
- def enum : Parser[EnumDef] = ENUM ~ ident ~ LBRACKET ~ enumDef.* ~ RBRACKET ~ typeAnnotations.? ^^ {
- case e ~ i ~ l ~ ed ~ r ~ t => EnumDef(i, ed.toList, t)
- }
-
- def enumDef : Parser[EnumValueDef] = ident ~ EQ ~ numericLit ~ typeAnnotations.? ~ commaOrSemicolon.? ^^ {
- case i ~ e ~ n ~ t ~ c => EnumValueDef(i, Some(IntConstant(n.toInt)), t)
- }
-
- def senum : Parser[SEnumDef] = SENUM ~ ident ~ LBRACKET ~ senumDef.* ~ RBRACKET ~ typeAnnotations.? ^^ {
- case se ~ i ~ l ~ sed ~ r ~ t => SEnumDef(i, sed.toList, t)
- }
-
- def senumDef : Parser[String] = stringLit <~ commaOrSemicolon.?
-
- def service : Parser[ServiceDef] = SERVICE ~ ident ~ extnds.? ~ LBRACKET ~ function.* ~
- RBRACKET ~ typeAnnotations.? ^^ {
- case s ~ i ~ e ~ lb ~ fs ~ rb ~ tA => ServiceDef(i, e, fs, tA)
- }
-
- def extnds : Parser[String] = EXTENDS ~> ident
-
- def function : Parser[FunctionDef] = ONEWAY.? ~ functionType ~ ident ~ LPAREN ~ field.* ~ RPAREN ~ throwz.? ~
- typeAnnotations.? ~ commaOrSemicolon.? ^^ {
- case o ~ fT ~ i ~ lp ~ fs ~ rp ~ th ~ tA ~ cS => FunctionDef(isOneWay(o), fT, i, fs, th, tA)
- }
-
- def throwz : Parser[List[FieldDef]] = THROWS ~ LPAREN ~ field.* ~ RPAREN ^^ {
- case t ~ l ~ fs ~ r => fs.toList
- }
-
- def functionType : Parser[FunctionType] = VOID ^^^ VoidType() | fieldType
-
- def xception : Parser[ExceptionDef] = EXCEPTION ~ ident ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ {
- case s ~ i ~ lb ~ fs ~ rb ~ tA => ExceptionDef(i, fs.toList, tA)
- }
-
- def union : Parser[UnionDef] = UNION ~ ident ~ XSD_ALL.? ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ {
- case s ~ i ~ xA ~ lb ~ fs ~ rb ~ tA => UnionDef(i, isXsdAll(xA), fs.toList, tA)
- }
-
- def struct : Parser[StructDef] = STRUCT ~ ident ~ XSD_ALL.? ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ {
- case s ~ i ~ xA ~ lb ~ fs ~ rb ~ tA => StructDef(i, isXsdAll(xA), fs.toList, tA)
- }
-
- def field : Parser[FieldDef] = fieldIdentifier.? ~ fieldRequiredness.? ~ fieldType ~ ident ~ fieldValue.? ~
- XSD_OPT.? ~ XSD_NILBLE.? ~ xsdAttributes.? ~ typeAnnotations.? ~ commaOrSemicolon.? ^^ {
- case fi ~ fr ~ ft ~id ~ fv ~ xo ~ xn ~ xa ~ tA ~ cS => FieldDef(
- fi,
- isRequired(fr),
- ft,
- id,
- fv,
- isXsdOptional(xo),
- isXsdNillable(xn),
- xa,
- tA
- )
- }
-
- def xsdAttributes : Parser[XsdAttributes] = XSD_ATTRS ~ LBRACKET ~ field.* ~ RBRACKET ^^ {
- case x ~ l ~ f ~ r => XsdAttributes(f)
- }
-
- def fieldValue = EQ ~> constValue
-
- def fieldRequiredness : Parser[String] = REQUIRED | OPTIONAL
-
- def fieldIdentifier : Parser[IntConstant] = numericLit <~ COLON ^^ {
- case n => IntConstant(n.toInt)
- }
-
- def const : Parser[ConstDef] = CONST ~ fieldType ~ ident ~ EQ ~ constValue ~ commaOrSemicolon.? ^^ {
- case c ~ fT ~ i ~ e ~ cV ~ cS => ConstDef(fT, i, cV)
- }
-
- def parseDouble(s: String) = try { Some(s.toDouble) } catch { case _ : Throwable => None }
-
- def constValue : Parser[ConstValue] = numericLit ^^ {
- case n => parseDouble(n) match {
- case Some(d) => DoubleConstant(d)
- case _ => IntConstant(n.toInt)
- }
- } |
- hexConstant ^^ { case h => IntConstant(Integer.parseInt(h, 16))} |
- stringLit ^^ { case s => StringConstant(s)} |
- ident ^^ { case i => IdConstant(i)} |
- constList |
- constMap
-
- def constValuePair = constValue ~ COLON ~ constValue ~ commaOrSemicolon.? ^^ {
- case k ~ c ~ v ~ cs => ConstantValuePair(k,v)
- }
-
- def constList = LSQBRACKET ~ (constValue <~ commaOrSemicolon).* ~ RSQBRACKET ^^ {
- case l ~ vs ~ r => ConstantList(vs)
- }
-
- def constMap = LBRACKET ~ constValuePair.* ~ RBRACKET ^^ {
- case l ~ ps ~ r => ConstantMap(ps)
- }
-}
-
-class ThriftLexer(val keywords: Seq[String], val delims : Seq[String]) extends StdLexical with ImplicitConversions {
-
- case class HexConstant(chars: String) extends Token {
- override def toString = chars
- }
-
- case class StIdentifier(chars: String) extends Token {
- override def toString = chars
- }
-
- reserved ++= keywords
-
- delimiters ++= delims
-
- override lazy val token: Parser[Token] =
- ( intConstant ^^ NumericLit
- | hexConstant ^^ HexConstant
- | dubConstant ^^ NumericLit
- | identifier ^^ processIdent
- | st_identifier ^^ StIdentifier
- | string ^^ StringLit
- | EofCh ^^^ EOF
- | '\'' ~> failure("unclosed string literal")
- | '"' ~> failure("unclosed string literal")
- | delim
- | failure("illegal character")
- )
-
- override def identChar = letter | elem('_')
-
- def identifier = identChar ~ (identChar | digit | '.' ).* ^^
- { case first ~ rest => (first :: rest).mkString }
-
- def st_identChar = letter | elem('-')
- def st_identifier = st_identChar ~ (st_identChar | digit | '.' | '_').* ^^
- { case first ~ rest => (first :: rest).mkString("")}
-
- override def whitespace: Parser[Any] =
- ( whitespaceChar
- | '/' ~ '*' ~ comment
- | '/' ~ '/' ~ chrExcept(EofCh, '\n').*
- | '#' ~ chrExcept(EofCh, '\n').*
- | '/' ~ '*' ~ failure("unclosed comment")
- ).*
-
- protected override def comment: Parser[Any] = (
- commentChar.* ~ '*' ~ '/'
- )
-
- protected def commentChar = chrExcept(EofCh, '*') | '*' ~ not('/')
-
- def string = '\"' ~> chrExcept('\"', '\n', EofCh).* <~ '\"' ^^ { _ mkString "" } |
- '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^ { _ mkString "" }
-
- def zero: Parser[String] = '0' ^^^ "0"
- def nonzero = elem("nonzero digit", d => d.isDigit && d != '0')
- def sign = elem("sign character", d => d == '-' || d == '+')
- def exponent = elem("exponent character", d => d == 'e' || d == 'E')
-
-
- def intConstant = opt(sign) ~> zero | intList
- def intList = opt(sign) ~ nonzero ~ rep(digit) ^^ {case s ~ x ~ y => (optString("", s) :: x :: y) mkString ""}
- def fracPart = '.' ~> rep(digit) ^^ { "." + _ mkString "" }
- def expPart = exponent ~ opt(sign) ~ rep1(digit) ^^ { case e ~ s ~ d =>
- e.toString + optString("", s) + d.mkString("")
- }
-
- def dubConstant = opt(sign) ~ digit.* ~ fracPart ~ opt(expPart) ^^ { case s ~ i ~ f ~ e =>
- optString("", s) + i + f + optString("", e)
- }
-
- val hexDigits = Set[Char]() ++ "0123456789abcdefABCDEF".toArray
- def hexDigit = elem("hex digit", hexDigits.contains(_))
-
- def hexConstant = '0' ~> 'x' ~> hexDigit.+ ^^ {case h => h.mkString("")}
-
-
- private def optString[A](pre: String, a: Option[A]) = a match {
- case Some(x) => pre + x.toString
- case None => ""
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftTypesGen.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftTypesGen.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftTypesGen.scala
deleted file mode 100755
index e04e192..0000000
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/thrift/ThriftTypesGen.scala
+++ /dev/null
@@ -1,316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.thrift
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.TypesDef
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.MetadataException
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.io.Source
-import scala.util.{Failure, Success, Try}
-
-
-case class CompositeRelation(typeName: String, fieldName: String, reverseFieldName: Option[String])
-
-/**
- * Convert a [[org.apache.hadoop.metadata.tools.thrift.ThriftDef ThriftDef]] to
- * [[TypesDef TypesDef]]. Currently there are several restrictions:
- *
- * - CppIncludes, SEnums are not allowed
- * - The only include allowed is that of "share/fb303/if/fb303.thrift". This include is ignored.
- * Any other include will trigger an exception
- * - Namespaces, TypeDefs, Contants, Unions, Exceptions, and Service definitions are ignored.
- * - So for fields typeDefs are not applied.
- * - Field Constant values are ignored.
- * - Type Annotations, XSD information is ignored.
- *
- * Thrift Structs can be mapped to Structs, Traits for Classes. The caller can specify their preference by
- * providing the structNames, classNames and thriftNames parameters. A Struct that is not in one of these 3
- * lists is not mapped.
- *
- * The ThriftDef doesn't specify if a relationship is composite. For e.g. in the thrift definition
- * {{{
- * struct Person {
- 1: string name,
- 2: Address addr,
- }
- struct Address {
- 1: string street,
- 2: string city,
- }
- * }}}
- *
- * If Person and Address are mapped to classes, you may not to make the Person -> Address a Composite relation.
- * The caller can specify these in the 'compositeRelations' parameter.
- *
- */
-class ThriftTypesGen(val structNames: List[String], val classNames: List[String], val traitNames: List[String],
- val compositeRelations: List[CompositeRelation]) {
-
- private val LOG: Logger = LoggerFactory.getLogger(classOf[ThriftTypesGen])
- private val FB_INCLUDE = "share/fb303/if/fb303.thrift"
- /**
- * for a (typeName, fieldName) specifies (isComposite, reverseName)
- * if entry doesn't exist than field is not composite.
- */
- private var compositeRelsMap: Map[(String, String), (Boolean, Option[String])] = Map()
-
- def apply(thriftResource: String): TypesDef = {
- val tDef = parseThrift(thriftResource)
-
- tDef.flatMap(buildCompositeRelations).flatMap(typesDef) match {
- case Success(t) => t
- case Failure(v) => throw v
- }
-
- }
-
- def buildCompositeRelations(thriftDef: ThriftDef): Try[ThriftDef] = Try {
-
- compositeRelations.foreach { cr =>
-
- val sDef = thriftDef.structs.find(_.name == cr.typeName)
- if (!sDef.isDefined) {
- throw new MetadataException(s"Unknown Struct (${cr.typeName}) specified in CompositeRelation")
-
- }
- val fDef = sDef.get.fields.find(_.name == cr.fieldName)
- if (!fDef.isDefined) {
- throw new MetadataException(s"Unknown Field (${cr.fieldName}) specified in CompositeRelation")
-
- }
-
- compositeRelsMap = compositeRelsMap + ((cr.typeName, cr.fieldName) ->(true, cr.reverseFieldName))
-
- if (cr.reverseFieldName.isDefined) {
- val reverseStructName = dataTypeName(fDef.get.fieldType)
- val reverseStructDef = thriftDef.structs.find(_.name == reverseStructName)
- if (!reverseStructDef.isDefined) {
- throw new MetadataException(s"Cannot find Struct $reverseStructName in CompositeRelation $cr")
- }
- val rfDef = reverseStructDef.get.fields.find(_.name == cr.reverseFieldName)
- if (!rfDef.isDefined) {
- throw new MetadataException(s"Unknown Reverse Field (${cr.reverseFieldName}) specified in CompositeRelation")
- }
-
- List(cr, CompositeRelation(reverseStructName, cr.reverseFieldName.get, Some(cr.fieldName)))
-
- compositeRelsMap = compositeRelsMap +
- ((reverseStructName, cr.reverseFieldName.get) ->(false, Some(cr.fieldName)))
- }
- }
-
- thriftDef
- }
-
- def typesDef(thriftDef: ThriftDef): Try[TypesDef] = {
- var tDef: Try[TypesDef] = Try {
- TypesDef(Seq(), Seq(), Seq(), Seq())
- }
-
- tDef.flatMap((t: TypesDef) => includes(t, thriftDef.includes)).flatMap((t: TypesDef) => cppIncludes(t, thriftDef.cppIncludes))
-
- tDef = tDef.flatMap((t: TypesDef) => includes(t, thriftDef.includes)).
- flatMap((t: TypesDef) => cppIncludes(t, thriftDef.cppIncludes)).
- flatMap((t: TypesDef) => namespaces(t, thriftDef.namespaces)).
- flatMap((t: TypesDef) => constants(t, thriftDef.constants)).
- flatMap((t: TypesDef) => senums(t, thriftDef.senums)).
- flatMap((t: TypesDef) => enums(t, thriftDef.enums)).
- flatMap((t: TypesDef) => structs(t, thriftDef.structs)).
- flatMap((t: TypesDef) => unions(t, thriftDef.unions)).
- flatMap((t: TypesDef) => exceptions(t, thriftDef.xceptions)).
- flatMap((t: TypesDef) => services(t, thriftDef.services))
-
-
- tDef
- }
-
- private def parseThrift(thriftResource: String): Try[ThriftDef] = {
- Try {
- LOG.debug("Parsing Thrift resource {}", thriftResource)
- val is = getClass().getResourceAsStream(thriftResource)
- val src: Source = Source.fromInputStream(is)
- val thriftStr: String = src.getLines().mkString("\n")
- val p = new ThriftParser
- var thriftDef: Option[ThriftDef] = p(thriftStr)
- thriftDef match {
- case Some(s) => s
- case None => {
- LOG.debug("Parse for thrift resource {} failed", thriftResource)
- throw new MetadataException(s"Failed to parse thrift resource: $thriftResource")
- }
- }
- }
- }
-
- @throws[MetadataException]
- private def dataTypeName(fT: FieldType): String = fT match {
- case IdentifierType(n) => n
- case BaseType(typ, _) => BASE_TYPES.toPrimitiveTypeName(typ)
- case ListType(elemType, _, _) => DataTypes.arrayTypeName(dataTypeName(elemType))
- case SetType(elemType, _, _) => DataTypes.arrayTypeName(dataTypeName(elemType))
- case MapType(keyType, valueType, _, _) => DataTypes.mapTypeName(dataTypeName(keyType), dataTypeName(valueType))
- }
-
- private def enumValue(e: EnumValueDef, defId: Int): EnumValue = e match {
- case EnumValueDef(value, Some(id), _) => new EnumValue(value, id.value)
- case EnumValueDef(value, None, _) => new EnumValue(value, defId)
- }
-
- private def enumDef(td: TypesDef, e: EnumDef): Try[TypesDef] = {
- Success(
- td.copy(enumTypes = td.enumTypes :+
- new EnumTypeDefinition(e.name, e.enumValues.zipWithIndex.map(t => enumValue(t._1, -t._2)): _*))
- )
- }
-
- private def includeDef(td: TypesDef, i: IncludeDef): Try[TypesDef] = {
- Try {
- if (i.value != FB_INCLUDE) {
- throw new MetadataException(s"Unsupported Include ${i.value}, only fb303.thrift is currently allowed.")
- }
- td
- }
- }
-
- private def cppIncludeDef(td: TypesDef, i: CppIncludeDef): Try[TypesDef] = {
- Try {
- throw new MetadataException(s"Unsupported CppInclude ${i.value}.")
- }
- }
-
- private def namespaceDef(td: TypesDef, i: NamespaceDef): Try[TypesDef] = {
- Try {
- LOG.debug(s"Ignoring Namespace definition $i")
- td
- }
- }
-
- private def constantDef(td: TypesDef, i: ConstDef): Try[TypesDef] = {
- Try {
- LOG.debug(s"Ignoring ConstantDef definition $i")
- td
- }
- }
-
- private def senumDef(td: TypesDef, i: SEnumDef): Try[TypesDef] = {
- Try {
- throw new MetadataException(s"Unsupported SEnums ${i}.")
- }
- }
-
- private def fieldDef(typName: String, fd: FieldDef): AttributeDefinition = {
- val name: String = fd.name
- val dTName: String = dataTypeName(fd.fieldType)
-
- var m: Multiplicity = Multiplicity.OPTIONAL
-
- if (fd.requiredNess) {
- m = Multiplicity.REQUIRED
- }
-
- fd.fieldType match {
- case _: ListType => m = Multiplicity.COLLECTION
- case _: SetType => m = Multiplicity.SET
- case _ => ()
- }
-
- var isComposite = false
- var reverseAttrName: String = null
-
- val r = compositeRelsMap.get((typName, name))
- if (r.isDefined) {
- isComposite = r.get._1
- if (r.get._2.isDefined) {
- reverseAttrName = r.get._2.get
- }
- }
- new AttributeDefinition(name, dTName, m, isComposite, reverseAttrName)
- }
-
- private def structDef(td: TypesDef, structDef: StructDef): Try[TypesDef] = Try {
- val typeName: String = structDef.name
-
- typeName match {
- case t if structNames contains t => td.copy(structTypes = td.structTypes :+
- new StructTypeDefinition(typeName, structDef.fields.map(fieldDef(typeName, _)).toArray))
- case t: String if traitNames contains t => {
- val ts = td.traitTypes :+
- new HierarchicalTypeDefinition[TraitType](classOf[TraitType],
- typeName, ImmutableList.of[String](), structDef.fields.map(fieldDef(typeName, _)).toArray)
- td.copy(traitTypes = ts)
- }
- case t: String if classNames contains t => {
- val cs = td.classTypes :+
- new HierarchicalTypeDefinition[ClassType](classOf[ClassType],
- typeName, ImmutableList.of[String](), structDef.fields.map(fieldDef(typeName, _)).toArray)
- td.copy(classTypes = cs)
- }
- case _ => td
- }
- }
-
- private def unionDef(td: TypesDef, i: UnionDef): Try[TypesDef] = {
- Try {
- LOG.debug(s"Ignoring Union definition $i")
- td
- }
- }
-
- private def exceptionDef(td: TypesDef, i: ExceptionDef): Try[TypesDef] = {
- Try {
- LOG.debug(s"Ignoring Exception definition $i")
- td
- }
- }
-
- private def serviceDef(td: TypesDef, i: ServiceDef): Try[TypesDef] = {
- Try {
- LOG.debug(s"Ignoring Service definition $i")
- td
- }
- }
-
- private def applyList[T](fn: (TypesDef, T) => Try[TypesDef])(td: TypesDef, l: List[T]): Try[TypesDef] = {
- l.foldLeft[Try[TypesDef]](Success(td))((b, a) => b.flatMap({ Unit => fn(td, a)}))
- }
-
- private def includes = applyList(includeDef) _
-
- private def cppIncludes = applyList(cppIncludeDef) _
-
- private def namespaces = applyList(namespaceDef) _
-
- private def constants = applyList(constantDef) _
-
- private def enums = applyList(enumDef) _
-
- private def senums = applyList(senumDef) _
-
- private def structs = applyList(structDef) _
-
- private def unions = applyList(unionDef) _
-
- private def exceptions = applyList(exceptionDef) _
-
- private def services = applyList(serviceDef) _
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/atlas/tools/dsl/DSLTest.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/atlas/tools/dsl/DSLTest.scala b/tools/src/test/scala/org/apache/atlas/tools/dsl/DSLTest.scala
new file mode 100755
index 0000000..2691600
--- /dev/null
+++ b/tools/src/test/scala/org/apache/atlas/tools/dsl/DSLTest.scala
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.dsl
+
+import org.apache.atlas.tools.hive.HiveMockMetadataService
+import org.apache.atlas.typesystem.types.utils.TypesUtil
+import org.apache.atlas.typesystem.types.{DataTypes, StructType, TypeSystem}
+import org.json4s.native.JsonMethods._
+import org.junit.{Assert, Before, Test}
+
+/**
+ * DSL Test.
+ */
+class DSLTest {
+ val STRUCT_TYPE_1: String = "t1"
+ val STRUCT_TYPE_2: String = "t2"
+
+
+ @Before
+ def setup {
+ val ts: TypeSystem = TypeSystem.getInstance
+ ts.reset()
+
+ val structType: StructType = ts.defineStructType(
+ STRUCT_TYPE_1, true,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+ TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+ TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+ TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+ TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+ TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+ TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+ TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ TypesUtil.createOptionalAttrDef("o",
+ ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+ val recursiveStructType: StructType = ts.defineStructType(
+ STRUCT_TYPE_2, true,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2))
+ }
+
+ @Test def test1 {
+
+ // 1. Existing Types in System
+ //Assert.assertEquals(s"${listTypes}", "[t2, t1, int, array<bigdecimal>, long, double, date, float, short, biginteger, byte, string, boolean, bigdecimal, map<string,double>, array<int>]")
+
+ defineStructType("mytype",
+ attrDef("a", INT_TYPE, ATTR_REQUIRED),
+ attrDef("b", BOOLEAN_TYPE),
+ attrDef("c", BYTE_TYPE),
+ attrDef("d", SHORT_TYPE),
+ attrDef("e", INT_TYPE),
+ attrDef("f", INT_TYPE),
+ attrDef("g", LONG_TYPE),
+ attrDef("h", FLOAT_TYPE),
+ attrDef("i", DOUBLE_TYPE),
+ attrDef("j", BIGINT_TYPE),
+ attrDef("k", BIGDECIMAL_TYPE),
+ attrDef("l", DATE_TYPE),
+ attrDef("m", arrayType(INT_TYPE)),
+ attrDef("n", arrayType(BIGDECIMAL_TYPE)),
+ attrDef("o", mapType(STRING_TYPE, DOUBLE_TYPE)))
+
+ // 2. 'mytype' available as a a Type
+ Assert.assertEquals(s"${listTypes}", "[array<bigdecimal>,array<int>,map<string,double>,mytype,t1,t2]")
+
+ // 3. Create a 'mytype' instance from Json
+ val i = createInstance("mytype", """
+ {
+ "$typeName$":"mytype",
+ "e":1,
+ "n":[1,1.1],
+ "h":1.0,
+ "b":true,
+ "k":1,
+ "j":1,
+ "d":2,
+ "m":[1,1],
+ "g":1,
+ "a":1,
+ "i":1.0,
+ "c":1,
+ "l":"2014-12-03T08:00:00.000Z",
+ "f":1,
+ "o":{
+ "b":2.0,
+ "a":1.0
+ }
+ }
+ """)
+
+ // 4. Navigate mytype instance in code
+ // Examples of Navigate mytype instance in code
+ Assert.assertEquals(s"${i.a}", "1")
+ Assert.assertEquals(s"${i.o}", "{b=2.0, a=1.0}")
+ Assert.assertEquals(s"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}", "[b, a]")
+
+ // 5. Serialize mytype instance to Json
+ Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03T08:00:00.000Z\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}")
+ }
+
+ @Test def test2 {
+
+ // 1. Existing Types in System
+ Assert.assertEquals(s"${listTypes}", "[array<bigdecimal>,array<int>,map<string,double>,t1,t2]")
+
+ val addrType = defineStructType("addressType",
+ attrDef("houseNum", INT_TYPE, ATTR_REQUIRED),
+ attrDef("street", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("city", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("state", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("zip", INT_TYPE, ATTR_REQUIRED),
+ attrDef("country", STRING_TYPE, ATTR_REQUIRED)
+ )
+
+ val personType = defineStructType("personType",
+ attrDef("first_name", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("last_name", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("address", addrType)
+ )
+
+ // 2. updated Types in System
+ Assert.assertEquals(s"${listTypes}", "[addressType,array<bigdecimal>,array<int>,map<string,double>,personType,t1,t2]")
+
+
+ // 3. Construct a Person in Code
+ val person = createInstance("personType")
+ val address = createInstance("addressType")
+
+ person.first_name = "Meta"
+ person.last_name = "Hadoop"
+
+ address.houseNum = 3460
+ address.street = "W Bayshore Rd"
+ address.city = "Palo Alto"
+ address.state = "CA"
+ address.zip = 94303
+ address.country = "USA"
+
+ person.address = address
+
+ // 4. Convert to Json
+ Assert.assertEquals(s"${pretty(render(person))}", "{\n \"$typeName$\":\"personType\",\n \"first_name\":\"Meta\",\n \"address\":{\n \"$typeName$\":\"addressType\",\n \"houseNum\":3460,\n \"city\":\"Palo Alto\",\n \"country\":\"USA\",\n \"state\":\"CA\",\n \"zip\":94303,\n \"street\":\"W Bayshore Rd\"\n },\n \"last_name\":\"Hadoop\"\n}");
+
+ val p2 = createInstance("personType", """{
+ "first_name":"Meta",
+ "address":{
+ "houseNum":3460,
+ "city":"Palo Alto",
+ "country":"USA",
+ "state":"CA",
+ "zip":94303,
+ "street":"W Bayshore Rd"
+ },
+ "last_name":"Hadoop"
+ }""")
+
+ }
+
+ @Test def testHive(): Unit = {
+ val hiveTable = HiveMockMetadataService.getTable("tpcds", "date_dim")
+ //println(hiveTable)
+
+ //name : String, typeName : String, comment : String
+ val fieldType = defineStructType("FieldSchema",
+ attrDef("name", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("typeName", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("comment", STRING_TYPE)
+ )
+ /*
+ SerDe(name : String, serializationLib : String, parameters : Map[String, String])
+ */
+ defineStructType("SerDe",
+ attrDef("name", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("serializationLib", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("parameters", mapType(STRING_TYPE, STRING_TYPE))
+ )
+
+ /*
+ StorageDescriptor(fields : List[FieldSchema],
+ location : String, inputFormat : String,
+ outputFormat : String, compressed : Boolean,
+ numBuckets : Int, bucketColumns : List[String],
+ sortColumns : List[String],
+ parameters : Map[String, String],
+ storedAsSubDirs : Boolean
+ )
+ */
+ val sdType = defineStructType("StorageDescriptor",
+ attrDef("location", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("inputFormat", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("outputFormat", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("compressed", BOOLEAN_TYPE),
+ attrDef("numBuckets", INT_TYPE),
+ attrDef("bucketColumns", arrayType(STRING_TYPE)),
+ attrDef("sortColumns", arrayType(STRING_TYPE)),
+ attrDef("parameters", mapType(STRING_TYPE, STRING_TYPE)),
+ attrDef("storedAsSubDirs", BOOLEAN_TYPE)
+ )
+
+ /*
+ case class Table(dbName : String, tableName : String, storageDesc : StorageDescriptor,
+ parameters : Map[String, String],
+ tableType : String)
+ */
+ defineStructType("Table",
+ attrDef("dbName", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("tableName", STRING_TYPE, ATTR_REQUIRED),
+ attrDef("storageDesc", sdType, ATTR_REQUIRED),
+ attrDef("compressed", BOOLEAN_TYPE),
+ attrDef("numBuckets", INT_TYPE),
+ attrDef("bucketColumns", arrayType(STRING_TYPE)),
+ attrDef("sortColumns", arrayType(STRING_TYPE)),
+ attrDef("parameters", mapType(STRING_TYPE, STRING_TYPE)),
+ attrDef("storedAsSubDirs", BOOLEAN_TYPE)
+ )
+ }
+}
[57/58] [abbrv] incubator-atlas git commit: Removed the footer
Posted by ve...@apache.org.
Removed the footer
Removed the footer
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/e426db2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/e426db2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/e426db2d
Branch: refs/heads/master
Commit: e426db2ddafda5071a6174f411323e37dbda1fd6
Parents: da9f12a
Author: DarshanKumar <da...@gmail.com>
Authored: Wed Jun 17 00:33:21 2015 +0530
Committer: DarshanKumar <da...@gmail.com>
Committed: Wed Jun 17 00:33:21 2015 +0530
----------------------------------------------------------------------
dashboard/v2/public/index.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/e426db2d/dashboard/v2/public/index.html
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/index.html b/dashboard/v2/public/index.html
index b5ab634..5ce839d 100644
--- a/dashboard/v2/public/index.html
+++ b/dashboard/v2/public/index.html
@@ -49,7 +49,7 @@
</div>
<footer class="footer navbar-bottom">
<div class="container">
- <p align="right">Powered by<img src="modules/home/img/logo-green.png"></p>
+ <!--<p align="right">Powered by<img src="modules/home/img/logo-green.png"></p>-->
</div>
</footer>
[26/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
new file mode 100755
index 0000000..642a8eb
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
@@ -0,0 +1,676 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.common.collect.ImmutableList;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Compare;
+import com.tinkerpop.blueprints.GraphQuery;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.BaseTest;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.commons.lang.RandomStringUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+import scala.actors.threadpool.Arrays;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * GraphBackedMetadataRepository test
+ *
+ * Guice loads the dependencies and injects the necessary objects
+ *
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class GraphBackedMetadataRepositoryTest {
+
+ private static final String ENTITY_TYPE = "Department";
+ private static final String DATABASE_TYPE = "hive_database";
+ private static final String DATABASE_NAME = "foo";
+ private static final String TABLE_TYPE = "hive_table";
+ private static final String TABLE_NAME = "bar";
+ private static final String CLASSIFICATION = "classification";
+ private static final String PII = "PII";
+ private static final String SUPER_TYPE_NAME = "Base";
+
+ @Inject
+ private GraphProvider<TitanGraph> graphProvider;
+
+ @Inject
+ private GraphBackedMetadataRepository repositoryService;
+
+ @Inject
+ private GraphBackedDiscoveryService discoveryService;
+
+ private TypeSystem typeSystem;
+ private String guid;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ typeSystem = TypeSystem.getInstance();
+ typeSystem.reset();
+
+ new GraphBackedSearchIndexer(graphProvider);
+
+ TestUtils.defineDeptEmployeeTypes(typeSystem);
+ createHiveTypes();
+ }
+
+/*
+ @AfterMethod
+ public void tearDown() throws Exception {
+ TestUtils.dumpGraph(graphProvider.get());
+ }
+*/
+
+ @Test
+ public void testSubmitEntity() throws Exception {
+ Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
+ ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+ guid = repositoryService.createEntity(hrDept2);
+ Assert.assertNotNull(guid);
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testGetEntityDefinitionForDepartment() throws Exception {
+ ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
+ Assert.assertNotNull(entity);
+ }
+
+ @Test (expectedExceptions = RepositoryException.class)
+ public void testGetEntityDefinitionNonExistent() throws Exception {
+ repositoryService.getEntityDefinition("blah");
+ Assert.fail();
+ }
+
+ @Test
+ public void testGetEntityList() throws Exception {
+ List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
+ System.out.println("entityList = " + entityList);
+ Assert.assertNotNull(entityList);
+ Assert.assertEquals(entityList.size(), 1); // one department
+ }
+
+ @Test
+ public void testGetTypeAttributeName() throws Exception {
+ Assert.assertEquals(
+ repositoryService.getTypeAttributeName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
+ }
+
+ @Test (dependsOnMethods = "testSubmitEntity")
+ public void testGetTraitLabel() throws Exception {
+ Assert.assertEquals(repositoryService.getTraitLabel(
+ typeSystem.getDataType(ClassType.class, TABLE_TYPE),
+ CLASSIFICATION), TABLE_TYPE + "." + CLASSIFICATION);
+ }
+
+ @Test
+ public void testCreateEntity() throws Exception {
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ databaseInstance.set("name", DATABASE_NAME);
+ databaseInstance.set("description", "foo database");
+ databaseInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
+
+ databaseInstance.set("namespace", "colo:cluster:hive:db");
+ databaseInstance.set("cluster", "cluster-1");
+ databaseInstance.set("colo", "colo-1");
+ System.out.println("databaseInstance = " + databaseInstance);
+
+ ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
+ ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
+ System.out.println("db = " + db);
+
+ String dbGUID = repositoryService.createEntity(db);
+ System.out.println("added db = " + dbGUID);
+
+ Referenceable dbInstance = new Referenceable(
+ dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
+
+ ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
+ String tableGUID = repositoryService.createEntity(table);
+ System.out.println("added table = " + tableGUID);
+ }
+
+ @Test(dependsOnMethods = "testCreateEntity")
+ public void testGetEntityDefinition() throws Exception {
+ String guid = getGUID();
+
+ ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
+ Assert.assertEquals(table.getDate("created"), new Date(BaseTest.TEST_DATE_IN_LONG));
+ System.out.println("*** table = " + table);
+ }
+
+ private String getGUID() {
+ Vertex tableVertex = getTableEntityVertex();
+
+ String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ if (guid == null) {
+ Assert.fail();
+ }
+ return guid;
+ }
+
+ private Vertex getTableEntityVertex() {
+ TitanGraph graph = graphProvider.get();
+ GraphQuery query = graph.query()
+ .has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
+ Iterator<Vertex> results = query.vertices().iterator();
+ // returning one since guid should be unique
+ Vertex tableVertex = results.hasNext() ? results.next() : null;
+ if (tableVertex == null) {
+ Assert.fail();
+ }
+
+ return tableVertex;
+ }
+
+ @Test (dependsOnMethods = "testCreateEntity")
+ public void testGetTraitNames() throws Exception {
+ final List<String> traitNames = repositoryService.getTraitNames(getGUID());
+ Assert.assertEquals(traitNames.size(), 1);
+ Assert.assertEquals(traitNames, Arrays.asList(new String[]{CLASSIFICATION}));
+ }
+
+ @Test
+ public void testGetTraitNamesForEmptyTraits() throws Exception {
+ final List<String> traitNames = repositoryService.getTraitNames(guid);
+ Assert.assertEquals(traitNames.size(), 0);
+ }
+
+ @Test (expectedExceptions = EntityNotFoundException.class)
+ public void testGetTraitNamesForBadEntity() throws Exception {
+ repositoryService.getTraitNames(UUID.randomUUID().toString());
+ Assert.fail();
+ }
+
+ @Test (dependsOnMethods = "testGetTraitNames")
+ public void testAddTrait() throws Exception {
+ final String aGUID = getGUID();
+
+ List<String> traitNames = repositoryService.getTraitNames(aGUID);
+ System.out.println("traitNames = " + traitNames);
+ Assert.assertEquals(traitNames.size(), 1);
+ Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+ Assert.assertFalse(traitNames.contains(PII));
+
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef(PII, ImmutableList.<String>of());
+ TraitType traitType = typeSystem.defineTraitType(piiTrait);
+ ITypedStruct traitInstance = traitType.createInstance();
+
+ repositoryService.addTrait(aGUID, traitInstance);
+
+ // refresh trait names
+ traitNames = repositoryService.getTraitNames(aGUID);
+ Assert.assertEquals(traitNames.size(), 2);
+ Assert.assertTrue(traitNames.contains(PII));
+ Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+ }
+
+ @Test (dependsOnMethods = "testAddTrait")
+ public void testAddTraitWithAttribute() throws Exception {
+ final String aGUID = getGUID();
+ final String traitName = "P_I_I";
+
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
+ TraitType traitType = typeSystem.defineTraitType(piiTrait);
+ ITypedStruct traitInstance = traitType.createInstance();
+ traitInstance.set("type", "SSN");
+
+ repositoryService.addTrait(aGUID, traitInstance);
+
+ TestUtils.dumpGraph(graphProvider.get());
+
+ // refresh trait names
+ List<String> traitNames = repositoryService.getTraitNames(aGUID);
+ Assert.assertEquals(traitNames.size(), 3);
+ Assert.assertTrue(traitNames.contains(traitName));
+
+ ITypedReferenceableInstance instance = repositoryService.getEntityDefinition(aGUID);
+ IStruct traitInstanceRef = instance.getTrait(traitName);
+ String type = (String) traitInstanceRef.get("type");
+ Assert.assertEquals(type, "SSN");
+ }
+
+ @Test (expectedExceptions = NullPointerException.class)
+ public void testAddTraitWithNullInstance() throws Exception {
+ repositoryService.addTrait(getGUID(), null);
+ Assert.fail();
+ }
+
+ @Test (dependsOnMethods = "testAddTrait", expectedExceptions = RepositoryException.class)
+ public void testAddTraitForBadEntity() throws Exception {
+ TraitType traitType = typeSystem.getDataType(TraitType.class, PII);
+ ITypedStruct traitInstance = traitType.createInstance();
+
+ repositoryService.addTrait(UUID.randomUUID().toString(), traitInstance);
+ Assert.fail();
+ }
+
+ @Test (dependsOnMethods = "testAddTrait")
+ public void testDeleteTrait() throws Exception {
+ final String aGUID = getGUID();
+
+ List<String> traitNames = repositoryService.getTraitNames(aGUID);
+ Assert.assertEquals(traitNames.size(), 3);
+ Assert.assertTrue(traitNames.contains(PII));
+ Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+ Assert.assertTrue(traitNames.contains("P_I_I"));
+
+ repositoryService.deleteTrait(aGUID, PII);
+
+ // refresh trait names
+ traitNames = repositoryService.getTraitNames(aGUID);
+ Assert.assertEquals(traitNames.size(), 2);
+ Assert.assertTrue(traitNames.contains(CLASSIFICATION));
+ Assert.assertFalse(traitNames.contains(PII));
+ }
+
+ @Test (expectedExceptions = RepositoryException.class)
+ public void testDeleteTraitForNonExistentEntity() throws Exception {
+ repositoryService.deleteTrait(UUID.randomUUID().toString(), PII);
+ Assert.fail();
+ }
+
+ @Test (expectedExceptions = RepositoryException.class)
+ public void testDeleteTraitForNonExistentTrait() throws Exception {
+ final String aGUID = getGUID();
+ repositoryService.deleteTrait(aGUID, "PCI");
+ Assert.fail();
+ }
+
+ @Test (dependsOnMethods = "testCreateEntity")
+ public void testGetIdFromVertex() throws Exception {
+ Vertex tableVertex = getTableEntityVertex();
+
+ String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ if (guid == null) {
+ Assert.fail();
+ }
+
+ Id expected = new Id(guid,
+ tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TABLE_TYPE);
+ Assert.assertEquals(repositoryService.getIdFromVertex(TABLE_TYPE, tableVertex), expected);
+ }
+
+ @Test (dependsOnMethods = "testCreateEntity")
+ public void testGetTypeName() throws Exception {
+ Vertex tableVertex = getTableEntityVertex();
+ Assert.assertEquals(repositoryService.getTypeName(tableVertex), TABLE_TYPE);
+ }
+
+ @Test(dependsOnMethods = "testCreateEntity")
+ public void testSearchByDSLQuery() throws Exception {
+ String dslQuery = "hive_database as PII";
+ System.out.println("Executing dslQuery = " + dslQuery);
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ Assert.assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ Assert.assertEquals(results.length(), 3);
+ System.out.println("results = " + results);
+
+ Object query = results.get("query");
+ Assert.assertNotNull(query);
+
+ JSONObject dataType = results.getJSONObject("dataType");
+ Assert.assertNotNull(dataType);
+ String typeName = dataType.getString("typeName");
+ Assert.assertNotNull(typeName);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertNotNull(rows);
+ Assert.assertTrue(rows.length() > 0);
+
+ for (int index = 0; index < rows.length(); index++) {
+ JSONObject row = rows.getJSONObject(index);
+ String type = row.getString("$typeName$");
+ Assert.assertEquals(type, "hive_database");
+
+ String name = row.getString("name");
+ Assert.assertEquals(name, DATABASE_NAME);
+ }
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testSearchByDSLWithInheritance() throws Exception {
+ String dslQuery = "Person where name = 'Jane'";
+ System.out.println("Executing dslQuery = " + dslQuery);
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ Assert.assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ Assert.assertEquals(results.length(), 3);
+ System.out.println("results = " + results);
+
+ Object query = results.get("query");
+ Assert.assertNotNull(query);
+
+ JSONObject dataType = results.getJSONObject("dataType");
+ Assert.assertNotNull(dataType);
+ String typeName = dataType.getString("typeName");
+ Assert.assertEquals(typeName, "Person");
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertEquals(rows.length(), 1);
+
+ JSONObject row = rows.getJSONObject(0);
+ Assert.assertEquals(row.getString("$typeName$"), "Manager");
+ Assert.assertEquals(row.getString("name"), "Jane");
+ }
+
+ @Test(dependsOnMethods = "testCreateEntity")
+ public void testBug37860() throws Exception {
+ String dslQuery =
+ "hive_table as t where name = 'bar' " +
+ "database where name = 'foo' and description = 'foo database' select t";
+ System.out.println("Executing dslQuery = " + dslQuery);
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ Assert.assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ Assert.assertEquals(results.length(), 3);
+ System.out.println("results = " + results);
+
+ Object query = results.get("query");
+ Assert.assertNotNull(query);
+
+ JSONObject dataType = results.getJSONObject("dataType");
+ Assert.assertNotNull(dataType);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertEquals(rows.length(), 1);
+
+ }
+
+ /**
+ * Full text search requires GraphBackedSearchIndexer, and GraphBackedSearchIndexer can't be enabled in
+ * GraphBackedDiscoveryServiceTest because of its test data. So, test for full text search is in
+ * GraphBackedMetadataRepositoryTest:(
+ */
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testFullTextSearch() throws Exception {
+ //todo fix this
+ //Weird: with lucene, the test passes without sleep
+ //but with elasticsearch, doesn't work without sleep. why??
+ long sleepInterval = 1000;
+
+ //person in hr department whose name is john
+ Thread.sleep(sleepInterval);
+ String response = discoveryService.searchByFullText("john");
+ Assert.assertNotNull(response);
+ JSONArray results = new JSONArray(response);
+ Assert.assertEquals(results.length(), 1);
+ JSONObject row = (JSONObject) results.get(0);
+ Assert.assertEquals(row.get("typeName"), "Person");
+
+ //person in hr department who lives in santa clara
+ response = discoveryService.searchByFullText("Jane AND santa AND clara");
+ Assert.assertNotNull(response);
+ results = new JSONArray(response);
+ Assert.assertEquals(results.length(), 1);
+ row = (JSONObject) results.get(0);
+ Assert.assertEquals(row.get("typeName"), "Manager");
+
+ //search for person in hr department whose name starts is john/jahn
+ response = discoveryService.searchByFullText("hr AND (john OR jahn)");
+ Assert.assertNotNull(response);
+ results = new JSONArray(response);
+ Assert.assertEquals(results.length(), 1);
+ row = (JSONObject) results.get(0);
+ Assert.assertEquals(row.get("typeName"), "Person");
+ }
+
+ private void createHiveTypes() throws Exception {
+ HierarchicalTypeDefinition<ClassType> superTypeDefinition =
+ TypesUtil.createClassTypeDef(SUPER_TYPE_NAME,
+ ImmutableList.<String>of(),
+ TypesUtil.createOptionalAttrDef("namespace", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("cluster", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("colo", DataTypes.STRING_TYPE));
+
+ HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
+ TypesUtil.createClassTypeDef(DATABASE_TYPE,
+ ImmutableList.of(SUPER_TYPE_NAME),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+
+
+ StructTypeDefinition structTypeDefinition =
+ new StructTypeDefinition("serdeType",
+ new AttributeDefinition[]{
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
+ });
+
+ EnumValue values[] = {
+ new EnumValue("MANAGED", 1),
+ new EnumValue("EXTERNAL", 2),
+ };
+
+ EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
+ typeSystem.defineEnumType(enumTypeDefinition);
+
+ HierarchicalTypeDefinition<ClassType> columnsDefinition =
+ TypesUtil.createClassTypeDef("column_type",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
+
+ StructTypeDefinition partitionDefinition =
+ new StructTypeDefinition("partition_type",
+ new AttributeDefinition[]{
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ });
+
+ HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
+ TypesUtil.createClassTypeDef(TABLE_TYPE,
+ ImmutableList.of(SUPER_TYPE_NAME),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
+ // enum
+ new AttributeDefinition("tableType", "tableType",
+ Multiplicity.REQUIRED, false, null),
+ // array of strings
+ new AttributeDefinition("columnNames",
+ String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
+ Multiplicity.COLLECTION, false, null),
+ // array of classes
+ new AttributeDefinition("columns",
+ String.format("array<%s>", "column_type"),
+ Multiplicity.COLLECTION, true, null),
+ // array of structs
+ new AttributeDefinition("partitions",
+ String.format("array<%s>", "partition_type"),
+ Multiplicity.COLLECTION, true, null),
+ // map of primitives
+ new AttributeDefinition("parametersMap",
+ DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
+ DataTypes.STRING_TYPE.getName()),
+ Multiplicity.COLLECTION, true, null),
+ // map of classes - todo - enable this
+// new AttributeDefinition("columnsMap",
+// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
+// "column_type"),
+// Multiplicity.COLLECTION, true, null),
+ // map of structs todo - enable this
+// new AttributeDefinition("partitionsMap",
+// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
+// "partition_type"),
+// Multiplicity.COLLECTION, true, null),
+ // struct reference
+ new AttributeDefinition("serde1",
+ "serdeType", Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("serde2",
+ "serdeType", Multiplicity.REQUIRED, false, null),
+ // class reference
+ new AttributeDefinition("database",
+ DATABASE_TYPE, Multiplicity.REQUIRED, true, null)
+ );
+
+ HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
+ TypesUtil.createTraitTypeDef(CLASSIFICATION,
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+
+ HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition =
+ TypesUtil.createTraitTypeDef("fetl" + CLASSIFICATION,
+ ImmutableList.of(CLASSIFICATION),
+ TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+
+ typeSystem.defineTypes(
+ ImmutableList.of(structTypeDefinition, partitionDefinition),
+ ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition),
+ ImmutableList.of(superTypeDefinition, databaseTypeDefinition,
+ columnsDefinition, tableTypeDefinition));
+ }
+
+ private ITypedReferenceableInstance createHiveTableInstance(
+ Referenceable databaseInstance) throws Exception {
+ Referenceable tableInstance = new Referenceable(TABLE_TYPE, CLASSIFICATION);
+ tableInstance.set("name", TABLE_NAME);
+ tableInstance.set("description", "bar table");
+ tableInstance.set("type", "managed");
+ tableInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
+ tableInstance.set("tableType", 1); // enum
+
+ // super type
+ tableInstance.set("namespace", "colo:cluster:hive:db:table");
+ tableInstance.set("cluster", "cluster-1");
+ tableInstance.set("colo", "colo-1");
+
+ // refer to an existing class
+ tableInstance.set("database", databaseInstance);
+
+ ArrayList<String> columnNames = new ArrayList<>();
+ columnNames.add("first_name");
+ columnNames.add("last_name");
+ tableInstance.set("columnNames", columnNames);
+
+ Struct traitInstance = (Struct) tableInstance.getTrait(CLASSIFICATION);
+ traitInstance.set("tag", "foundation_etl");
+
+ Struct serde1Instance = new Struct("serdeType");
+ serde1Instance.set("name", "serde1");
+ serde1Instance.set("serde", "serde1");
+ tableInstance.set("serde1", serde1Instance);
+
+ Struct serde2Instance = new Struct("serdeType");
+ serde2Instance.set("name", "serde2");
+ serde2Instance.set("serde", "serde2");
+ tableInstance.set("serde2", serde2Instance);
+
+ // HashMap<String, Referenceable> columnsMap = new HashMap<>();
+ ArrayList<Referenceable> columns = new ArrayList<>();
+ for (int index = 0; index < 5; index++) {
+ Referenceable columnInstance = new Referenceable("column_type");
+ final String name = "column_" + index;
+ columnInstance.set("name", name);
+ columnInstance.set("type", "string");
+
+ columns.add(columnInstance);
+ // columnsMap.put(name, columnInstance);
+ }
+ tableInstance.set("columns", columns);
+ // tableInstance.set("columnsMap", columnsMap);
+
+// HashMap<String, Struct> partitionsMap = new HashMap<>();
+ ArrayList<Struct> partitions = new ArrayList<>();
+ for (int index = 0; index < 5; index++) {
+ Struct partitionInstance = new Struct("partition_type");
+ final String name = "partition_" + index;
+ partitionInstance.set("name", name);
+
+ partitions.add(partitionInstance);
+// partitionsMap.put(name, partitionInstance);
+ }
+ tableInstance.set("partitions", partitions);
+// tableInstance.set("partitionsMap", partitionsMap);
+
+ HashMap<String, String> parametersMap = new HashMap<>();
+ parametersMap.put("foo", "bar");
+ parametersMap.put("bar", "baz");
+ parametersMap.put("some", "thing");
+ tableInstance.set("parametersMap", parametersMap);
+
+ ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
+ return tableType.convert(tableInstance, Multiplicity.REQUIRED);
+ }
+
+ private String random() {
+ return RandomStringUtils.random(10);
+ }
+
+ @Test
+ public void testUTFValues() throws Exception {
+ Referenceable hrDept = new Referenceable("Department");
+ Referenceable john = new Referenceable("Person");
+ john.set("name", random());
+ john.set("department", hrDept);
+
+ hrDept.set("name", random());
+ hrDept.set("employees", ImmutableList.of(john));
+
+ ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+ guid = repositoryService.createEntity(hrDept2);
+ Assert.assertNotNull(guid);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
new file mode 100755
index 0000000..3f0782b
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.common.collect.ImmutableList;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanIndexQuery;
+import com.tinkerpop.blueprints.Compare;
+import com.tinkerpop.blueprints.GraphQuery;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Map;
+
+@Test
+@Guice(modules = RepositoryMetadataModule.class)
+public class GraphRepoMapperScaleTest {
+
+ private static final String DATABASE_TYPE = "hive_database_type";
+ private static final String DATABASE_NAME = "foo";
+ private static final String TABLE_TYPE = "hive_table_type";
+ private static final String TABLE_NAME = "bar";
+
+ @Inject
+ private GraphProvider<TitanGraph> graphProvider;
+ @Inject
+ private GraphBackedMetadataRepository repositoryService;
+
+ private GraphBackedSearchIndexer searchIndexer;
+ private TypeSystem typeSystem;
+ private String dbGUID;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ searchIndexer = new GraphBackedSearchIndexer(graphProvider);
+
+ typeSystem = TypeSystem.getInstance();
+
+ createHiveTypes();
+ }
+
+ @Test
+ public void testSubmitEntity() throws Exception {
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ databaseInstance.set("name", DATABASE_NAME);
+ databaseInstance.set("description", "foo database");
+ // System.out.println("databaseInstance = " + databaseInstance);
+
+ ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
+ ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
+
+ dbGUID = repositoryService.createEntity(db);
+
+ Referenceable dbInstance = new Referenceable(
+ dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
+
+ for (int index = 0; index < 1000; index++) {
+ ITypedReferenceableInstance table = createHiveTableInstance(dbInstance, index);
+ repositoryService.createEntity(table);
+ }
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testSearchIndex() throws Exception {
+ searchWithOutIndex(Constants.GUID_PROPERTY_KEY, dbGUID);
+ searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, "hive_column_type");
+ searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, TABLE_TYPE);
+
+ searchWithOutIndex("hive_table_type.name", "bar-999");
+ searchWithIndex("hive_table_type.name", "bar-999");
+
+ for (int index = 500; index < 600; index++) {
+ searchWithIndex("hive_table_type.name", "bar-" + index);
+ }
+ }
+
+ private void searchWithOutIndex(String key, String value) {
+ TitanGraph graph = graphProvider.get();
+ long start = System.currentTimeMillis();
+ int count = 0;
+ try {
+ GraphQuery query = graph.query()
+ .has(key, Compare.EQUAL, value);
+ for (Vertex ignored : query.vertices()) {
+ count++;
+ }
+ } finally {
+ System.out.println("Search on [" + key + "=" + value + "] returned results: " + count
+ + ", took " + (System.currentTimeMillis() - start) + " ms");
+ }
+ }
+
+ private void searchWithIndex(String key, String value) {
+ TitanGraph graph = graphProvider.get();
+ long start = System.currentTimeMillis();
+ int count = 0;
+ try {
+ String queryString = "v.\"" + key + "\":(" + value + ")";
+ TitanIndexQuery query = graph.indexQuery(Constants.VERTEX_INDEX, queryString);
+ for (TitanIndexQuery.Result<Vertex> ignored : query.vertices()) {
+ count++;
+ }
+ } finally {
+ System.out.println("Search on [" + key + "=" + value + "] returned results: " + count
+ + ", took " + (System.currentTimeMillis() - start) + " ms");
+ }
+ }
+
+ private void createHiveTypes() throws Exception {
+ HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
+ TypesUtil.createClassTypeDef(DATABASE_TYPE,
+ ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+
+ StructTypeDefinition structTypeDefinition =
+ new StructTypeDefinition("hive_serde_type",
+ new AttributeDefinition[]{
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
+ });
+
+ EnumValue values[] = {
+ new EnumValue("MANAGED", 1),
+ new EnumValue("EXTERNAL", 2),
+ };
+
+ EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("table_type", values);
+ EnumType enumType = typeSystem.defineEnumType(enumTypeDefinition);
+ searchIndexer.onAdd("table_type", enumType);
+
+ HierarchicalTypeDefinition<ClassType> columnsDefinition =
+ TypesUtil.createClassTypeDef("hive_column_type",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
+
+ StructTypeDefinition partitionDefinition =
+ new StructTypeDefinition("hive_partition_type",
+ new AttributeDefinition[]{
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ });
+
+ HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
+ TypesUtil.createClassTypeDef(TABLE_TYPE,
+ ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+ // enum
+ new AttributeDefinition("tableType", "table_type",
+ Multiplicity.REQUIRED, false, null),
+ // array of strings
+ new AttributeDefinition("columnNames",
+ String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
+ Multiplicity.COLLECTION, false, null),
+ // array of classes
+ new AttributeDefinition("columns",
+ String.format("array<%s>", "hive_column_type"),
+ Multiplicity.COLLECTION, true, null),
+ // array of structs
+ new AttributeDefinition("partitions",
+ String.format("array<%s>", "hive_partition_type"),
+ Multiplicity.COLLECTION, true, null),
+ // struct reference
+ new AttributeDefinition("serde1",
+ "hive_serde_type", Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("serde2",
+ "hive_serde_type", Multiplicity.REQUIRED, false, null),
+ // class reference
+ new AttributeDefinition("database",
+ DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
+
+ HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
+ TypesUtil.createTraitTypeDef("pii_type", ImmutableList.<String>of());
+
+ Map<String, IDataType> types = typeSystem.defineTypes(
+ ImmutableList.of(structTypeDefinition, partitionDefinition),
+ ImmutableList.of(classificationTypeDefinition),
+ ImmutableList.of(databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
+
+ for (Map.Entry<String, IDataType> entry : types.entrySet()) {
+ searchIndexer.onAdd(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private ITypedReferenceableInstance createHiveTableInstance(
+ Referenceable databaseInstance, int uberIndex) throws Exception {
+
+ Referenceable tableInstance = new Referenceable(TABLE_TYPE, "pii_type");
+ tableInstance.set("name", TABLE_NAME + "-" + uberIndex);
+ tableInstance.set("description", "bar table" + "-" + uberIndex);
+ tableInstance.set("type", "managed");
+ tableInstance.set("tableType", 1); // enum
+
+ // refer to an existing class
+ tableInstance.set("database", databaseInstance);
+
+ ArrayList<String> columnNames = new ArrayList<>();
+ columnNames.add("first_name" + "-" + uberIndex);
+ columnNames.add("last_name" + "-" + uberIndex);
+ tableInstance.set("columnNames", columnNames);
+
+ Struct serde1Instance = new Struct("hive_serde_type");
+ serde1Instance.set("name", "serde1" + "-" + uberIndex);
+ serde1Instance.set("serde", "serde1" + "-" + uberIndex);
+ tableInstance.set("serde1", serde1Instance);
+
+ Struct serde2Instance = new Struct("hive_serde_type");
+ serde2Instance.set("name", "serde2" + "-" + uberIndex);
+ serde2Instance.set("serde", "serde2" + "-" + uberIndex);
+ tableInstance.set("serde2", serde2Instance);
+
+ ArrayList<Referenceable> columns = new ArrayList<>();
+ for (int index = 0; index < 5; index++) {
+ Referenceable columnInstance = new Referenceable("hive_column_type");
+ columnInstance.set("name", "column_" + "-" + uberIndex + "-" + index);
+ columnInstance.set("type", "string");
+ columns.add(columnInstance);
+ }
+ tableInstance.set("columns", columns);
+
+ ArrayList<Struct> partitions = new ArrayList<>();
+ for (int index = 0; index < 5; index++) {
+ Struct partitionInstance = new Struct("hive_partition_type");
+ partitionInstance.set("name", "partition_" + "-" + uberIndex + "-" + index);
+ partitions.add(partitionInstance);
+ }
+ tableInstance.set("partitions", partitions);
+
+ ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
+ return tableType.convert(tableInstance, Multiplicity.REQUIRED);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/memory/ClassTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/memory/ClassTest.java b/repository/src/test/java/org/apache/atlas/repository/memory/ClassTest.java
new file mode 100755
index 0000000..113a975
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/memory/ClassTest.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.BaseTest;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ClassTest extends BaseTest {
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @Test
+ public void test1() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+
+ defineDeptEmployeeTypes(ts);
+ Referenceable hrDept = createDeptEg1(ts);
+ ClassType deptType = ts.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+
+ Assert.assertEquals(hrDept2.toString(), "{\n" +
+ "\tid : (type: Department, id: <unassigned>)\n" +
+ "\tname : \thr\n" +
+ "\temployees : \t[{\n" +
+ "\tid : (type: Person, id: <unassigned>)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : (type: Manager, id: <unassigned>)\n" +
+ "}, {\n" +
+ "\tid : (type: Manager, id: <unassigned>)\n" +
+ "\tsubordinates : \t[{\n" +
+ "\tid : (type: Person, id: <unassigned>)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : (type: Manager, id: <unassigned>)\n" +
+ "}]\n" +
+ "\tname : \tJane\n" +
+ "\tdepartment : (type: Department, id: <unassigned>)\n" +
+ "\tmanager : <null>\n" +
+ "\n" +
+ "\tSecurityClearance : \t{\n" +
+ "\t\tlevel : \t\t1\n" +
+ "\t}}]\n" +
+ "}");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/memory/EnumTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/memory/EnumTest.java b/repository/src/test/java/org/apache/atlas/repository/memory/EnumTest.java
new file mode 100755
index 0000000..731bca6
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/memory/EnumTest.java
@@ -0,0 +1,315 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.BaseTest;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.json.Serialization$;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.Map;
+
+public class EnumTest extends BaseTest {
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ void defineEnums(TypeSystem ts) throws MetadataException {
+ ts.defineEnumType("HiveObjectType",
+ new EnumValue("GLOBAL", 1),
+ new EnumValue("DATABASE", 2),
+ new EnumValue("TABLE", 3),
+ new EnumValue("PARTITION", 4),
+ new EnumValue("COLUMN", 5));
+
+ ts.defineEnumType("PrincipalType",
+ new EnumValue("USER", 1),
+ new EnumValue("ROLE", 2),
+ new EnumValue("GROUP", 3));
+
+ ts.defineEnumType("TxnState",
+ new EnumValue("COMMITTED", 1),
+ new EnumValue("ABORTED", 2),
+ new EnumValue("OPEN", 3));
+
+ ts.defineEnumType("LockLevel",
+ new EnumValue("DB", 1),
+ new EnumValue("TABLE", 2),
+ new EnumValue("PARTITION", 3));
+
+ }
+
+ protected void fillStruct(Struct s) throws MetadataException {
+ s.set("a", 1);
+ s.set("b", true);
+ s.set("c", (byte) 1);
+ s.set("d", (short) 2);
+ s.set("e", 1);
+ s.set("f", 1);
+ s.set("g", 1L);
+ s.set("h", 1.0f);
+ s.set("i", 1.0);
+ s.set("j", BigInteger.valueOf(1L));
+ s.set("k", new BigDecimal(1));
+ s.set("l", new Date(1418265358440L));
+ s.set("m", Lists.asList(1, new Integer[]{1}));
+ s.set("n",
+ Lists.asList(BigDecimal.valueOf(1.1), new BigDecimal[]{BigDecimal.valueOf(1.1)}));
+ Map<String, Double> hm = Maps.newHashMap();
+ hm.put("a", 1.0);
+ hm.put("b", 2.0);
+ s.set("o", hm);
+ s.set("enum1", "GLOBAL");
+ s.set("enum2", 1);
+ s.set("enum3", "COMMITTED");
+ s.set("enum4", 3);
+ }
+
+ protected Struct createStructWithEnum(String typeName) throws MetadataException {
+ Struct s = new Struct(typeName);
+ fillStruct(s);
+ return s;
+ }
+
+ protected Referenceable createInstanceWithEnum(String typeName) throws MetadataException {
+ Referenceable r = new Referenceable(typeName);
+ fillStruct(r);
+ return r;
+ }
+
+ protected ClassType defineClassTypeWithEnum(TypeSystem ts) throws MetadataException {
+ return ts.defineClassType(TypesUtil.createClassTypeDef("t4",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+ TypesUtil.createOptionalAttrDef("enum1",
+ ts.getDataType(EnumType.class, "HiveObjectType")),
+ TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+ TypesUtil.createOptionalAttrDef("enum2",
+ ts.getDataType(EnumType.class, "PrincipalType")),
+ TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+ TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+ TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+ TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+ TypesUtil
+ .createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
+ TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+ TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ TypesUtil.createOptionalAttrDef("o",
+ ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ TypesUtil.createOptionalAttrDef("enum4",
+ ts.getDataType(EnumType.class, "LockLevel"))));
+ }
+
+ @Test
+ public void testStruct() throws MetadataException {
+ TypeSystem ts = getTypeSystem();
+ defineEnums(ts);
+ StructType structType = ts.defineStructType("t3",
+ true,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+ TypesUtil.createOptionalAttrDef("enum1",
+ ts.getDataType(EnumType.class, "HiveObjectType")),
+ TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+ TypesUtil.createOptionalAttrDef("enum2",
+ ts.getDataType(EnumType.class, "PrincipalType")),
+ TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+ TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+ TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+ TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+ TypesUtil
+ .createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
+
+ TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+ TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ TypesUtil.createOptionalAttrDef("o",
+ ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ TypesUtil.createOptionalAttrDef("enum4",
+ ts.getDataType(EnumType.class, "LockLevel")));
+
+ Struct s = createStructWithEnum("t3");
+ ITypedStruct typedS = structType.convert(s, Multiplicity.REQUIRED);
+ Assert.assertEquals(typedS.toString(), "{\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t2\n" +
+ "\tenum1 : \tGLOBAL\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\tenum2 : \tUSER\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tenum3 : \tCOMMITTED\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.1, 1.1]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "\tenum4 : \tPARTITION\n" +
+ "}");
+ }
+
+ @Test
+ public void testClass() throws MetadataException {
+ TypeSystem ts = getTypeSystem();
+ defineEnums(ts);
+ ClassType clsType = defineClassTypeWithEnum(ts);
+
+ IReferenceableInstance r = createInstanceWithEnum("t4");
+ ITypedReferenceableInstance typedR = clsType.convert(r, Multiplicity.REQUIRED);
+ Assert.assertEquals(typedR.toString(), "{\n" +
+ "\tid : (type: t4, id: <unassigned>)\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t2\n" +
+ "\tenum1 : \tGLOBAL\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\tenum2 : \tUSER\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tenum3 : \tCOMMITTED\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.1, 1.1]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "\tenum4 : \tPARTITION\n" +
+ "}");
+ }
+
+ @Test
+ public void testStorage() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+ defineEnums(ts);
+ ClassType clsType = defineClassTypeWithEnum(ts);
+
+ getRepository().defineTypes(ImmutableList.of((HierarchicalType) clsType));
+
+ IReferenceableInstance r = createInstanceWithEnum("t4");
+ IReferenceableInstance r1 = getRepository().create(r);
+
+ ITypedReferenceableInstance r2 = getRepository().get(r1.getId());
+ Assert.assertEquals(r2.toString(), "{\n" +
+ "\tid : (type: t4, id: 1)\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t0\n" +
+ "\tenum1 : \tGLOBAL\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\tenum2 : \tUSER\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tenum3 : \tCOMMITTED\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.1, 1.1]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "\tenum4 : \tPARTITION\n" +
+ "}");
+ }
+
+ @Test
+ public void testJson() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+ defineEnums(ts);
+ ClassType clsType = defineClassTypeWithEnum(ts);
+
+ getRepository().defineTypes(ImmutableList.of((HierarchicalType) clsType));
+
+ IReferenceableInstance r = createInstanceWithEnum("t4");
+ IReferenceableInstance r1 = getRepository().create(r);
+
+ ITypedReferenceableInstance r2 = getRepository().get(r1.getId());
+ String jsonStr = Serialization$.MODULE$.toJson(r2);
+
+ IReferenceableInstance r3 = Serialization$.MODULE$.fromJson(jsonStr);
+ Assert.assertEquals(r3.toString(), "{\n" +
+ "\tid : (type: t4, id: 1)\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t0\n" +
+ "\tenum1 : \tGLOBAL\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\tenum2 : \tUSER\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tenum3 : \tCOMMITTED\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
+ ".100000000000000088817841970012523233890533447265625]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "\tenum4 : \tPARTITION\n" +
+ "}");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/memory/InstanceE2ETest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/memory/InstanceE2ETest.java b/repository/src/test/java/org/apache/atlas/repository/memory/InstanceE2ETest.java
new file mode 100755
index 0000000..29d3f65
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/memory/InstanceE2ETest.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.BaseTest;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization$;
+import org.apache.atlas.typesystem.json.Serialization$;
+import org.apache.atlas.typesystem.json.TypesSerialization$;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class InstanceE2ETest extends BaseTest {
+
+ protected List<HierarchicalTypeDefinition> createHiveTypes(TypeSystem typeSystem)
+ throws MetadataException {
+ ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
+
+ HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
+ TypesUtil.createClassTypeDef("hive_database",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+ typeDefinitions.add(databaseTypeDefinition);
+
+ HierarchicalTypeDefinition<ClassType> tableTypeDefinition = TypesUtil.createClassTypeDef(
+ "hive_table",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+ new AttributeDefinition("hive_database",
+ "hive_database", Multiplicity.REQUIRED, false, "hive_database"));
+ typeDefinitions.add(tableTypeDefinition);
+
+ HierarchicalTypeDefinition<TraitType> fetlTypeDefinition = TypesUtil.createTraitTypeDef(
+ "hive_fetl",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
+ typeDefinitions.add(fetlTypeDefinition);
+
+ typeSystem.defineTypes(
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(fetlTypeDefinition),
+ ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
+
+ return typeDefinitions;
+ }
+
+ protected Referenceable createHiveTableReferenceable()
+ throws MetadataException {
+ Referenceable databaseInstance = new Referenceable("hive_database");
+ databaseInstance.set("name", "hive_database");
+ databaseInstance.set("description", "foo database");
+
+ Referenceable tableInstance = new Referenceable("hive_table", "hive_fetl");
+ tableInstance.set("name", "t1");
+ tableInstance.set("description", "bar table");
+ tableInstance.set("type", "managed");
+ tableInstance.set("hive_database", databaseInstance);
+
+ Struct traitInstance = (Struct) tableInstance.getTrait("hive_fetl");
+ traitInstance.set("level", 1);
+
+ tableInstance.set("hive_fetl", traitInstance);
+
+ return tableInstance;
+ }
+
+ protected ITypedReferenceableInstance createHiveTableInstance(TypeSystem typeSystem)
+ throws MetadataException {
+ ClassType tableType = typeSystem.getDataType(ClassType.class, "hive_table");
+ return tableType.convert(createHiveTableReferenceable(), Multiplicity.REQUIRED);
+ }
+
+ @Test
+ public void testType() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+
+ createHiveTypes(ts);
+
+ String jsonStr = TypesSerialization$.MODULE$
+ .toJson(ts, ImmutableList.of("hive_database", "hive_table"));
+ System.out.println(jsonStr);
+
+ TypesDef typesDef1 = TypesSerialization$.MODULE$.fromJson(jsonStr);
+ System.out.println(typesDef1);
+
+ ts.reset();
+ ts.defineTypes(typesDef1);
+ jsonStr = TypesSerialization$.MODULE$
+ .toJson(ts, ImmutableList.of("hive_database", "hive_table"));
+ System.out.println(jsonStr);
+
+ }
+
+ @Test
+ public void testInstance() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+
+ createHiveTypes(ts);
+
+ ITypedReferenceableInstance i = createHiveTableInstance(getTypeSystem());
+
+ String jsonStr = Serialization$.MODULE$.toJson(i);
+ System.out.println(jsonStr);
+
+ i = Serialization$.MODULE$.fromJson(jsonStr);
+ System.out.println(i);
+ }
+
+ @Test
+ public void testInstanceSerialization() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+
+ createHiveTypes(ts);
+
+ Referenceable r = createHiveTableReferenceable();
+ String jsonStr = InstanceSerialization$.MODULE$.toJson(r, true);
+ Referenceable r1 = InstanceSerialization$.MODULE$.fromJsonReferenceable(jsonStr, true);
+ ClassType tableType = ts.getDataType(ClassType.class, "hive_table");
+
+ ITypedReferenceableInstance i = tableType.convert(r1, Multiplicity.REQUIRED);
+
+ jsonStr = Serialization$.MODULE$.toJson(i);
+ System.out.println(jsonStr);
+
+ i = Serialization$.MODULE$.fromJson(jsonStr);
+ System.out.println(i);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/memory/StorageTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/memory/StorageTest.java b/repository/src/test/java/org/apache/atlas/repository/memory/StorageTest.java
new file mode 100755
index 0000000..5b1159c
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/memory/StorageTest.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.BaseTest;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StorageTest extends BaseTest {
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @Test
+ public void test1() throws MetadataException {
+
+ TypeSystem ts = getTypeSystem();
+
+ defineDeptEmployeeTypes(ts);
+
+ Referenceable hrDept = createDeptEg1(ts);
+ ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
+ ITypedReferenceableInstance hrDept3 = getRepository().get(hrDept2.getId());
+ Assert.assertEquals(hrDept3.toString(), "{\n" +
+ "\tid : (type: Department, id: 1)\n" +
+ "\tname : \thr\n" +
+ "\temployees : \t[{\n" +
+ "\tid : (type: Person, id: 2)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: 1)\n" +
+ "\tmanager : (type: Manager, id: 3)\n" +
+ "}, {\n" +
+ "\tid : (type: Manager, id: 3)\n" +
+ "\tsubordinates : \t[(type: Person, id: 2)]\n" +
+ "\tname : \tJane\n" +
+ "\tdepartment : (type: Department, id: 1)\n" +
+ "\tmanager : <null>\n" +
+ "\n" +
+ "\tSecurityClearance : \t{\n" +
+ "\t\tlevel : \t\t1\n" +
+ "\t}}]\n" +
+ "}");
+ }
+
+ @Test
+ public void testGetPerson() throws MetadataException {
+ TypeSystem ts = getTypeSystem();
+ defineDeptEmployeeTypes(ts);
+
+ Referenceable hrDept = createDeptEg1(ts);
+ ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
+
+ Id e1Id = new Id(2, 0, "Person");
+ ITypedReferenceableInstance e1 = getRepository().get(e1Id);
+ Assert.assertEquals(e1.toString(), "{\n" +
+ "\tid : (type: Person, id: 2)\n" +
+ "\tname : \tJohn\n" +
+ "\tdepartment : (type: Department, id: 1)\n" +
+ "\tmanager : (type: Manager, id: 3)\n" +
+ "}");
+ }
+
+ @Test
+ public void testInvalidTypeName() throws MetadataException {
+ TypeSystem ts = getTypeSystem();
+ defineDeptEmployeeTypes(ts);
+
+ Referenceable hrDept = createDeptEg1(ts);
+ ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
+
+ Id e1Id = new Id(3, 0, "Person");
+ try {
+ ITypedReferenceableInstance e1 = getRepository().get(e1Id);
+ } catch (RepositoryException re) {
+ RepositoryException me = (RepositoryException) re.getCause();
+ Assert.assertEquals(me.getMessage(), "Invalid Id (unknown) : (type: Person, id: 3)");
+
+ }
+ }
+
+ @Test
+ public void testGetManager() throws MetadataException {
+ TypeSystem ts = getTypeSystem();
+ defineDeptEmployeeTypes(ts);
+
+ Referenceable hrDept = createDeptEg1(ts);
+ ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
+
+ Id m1Id = new Id(3, 0, "Manager");
+ ITypedReferenceableInstance m1 = getRepository().get(m1Id);
+ Assert.assertEquals(m1.toString(), "{\n" +
+ "\tid : (type: Manager, id: 3)\n" +
+ "\tsubordinates : \t[(type: Person, id: 2)]\n" +
+ "\tname : \tJane\n" +
+ "\tdepartment : (type: Department, id: 1)\n" +
+ "\tmanager : <null>\n" +
+ "\n" +
+ "\tSecurityClearance : \t{\n" +
+ "\t\tlevel : \t\t1\n" +
+ "\t}}");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/memory/StructTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/memory/StructTest.java b/repository/src/test/java/org/apache/atlas/repository/memory/StructTest.java
new file mode 100755
index 0000000..d42a899
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/memory/StructTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.BaseTest;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.json.InstanceSerialization$;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StructTest extends BaseTest {
+
+ StructType structType;
+ StructType recursiveStructType;
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ structType = (StructType) getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
+ recursiveStructType = (StructType) getTypeSystem()
+ .getDataType(StructType.class, STRUCT_TYPE_2);
+ }
+
+ @Test
+ public void test1() throws MetadataException {
+ Struct s = createStruct();
+ ITypedStruct ts = structType.convert(s, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t2\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.1, 1.1]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "}");
+ }
+
+ @Test
+ public void testRecursive() throws MetadataException {
+ Struct s1 = new Struct(recursiveStructType.getName());
+ s1.set("a", 1);
+ Struct s2 = new Struct(recursiveStructType.getName());
+ s2.set("a", 1);
+ s2.set("s", s1);
+ ITypedStruct ts = recursiveStructType.convert(s2, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\ta : \t1\n" +
+ "\ts : \t{\n" +
+ "\t\ta : \t\t1\n" +
+ "\t\ts : <null>\n" +
+ "\n" +
+ "\t}\n" +
+ "}");
+ }
+
+ @Test
+ public void testSerialization() throws MetadataException {
+ Struct s = createStruct();
+ String jsonStr = InstanceSerialization$.MODULE$.toJson(s, true);
+ Struct s1 = InstanceSerialization$.MODULE$.fromJsonStruct(jsonStr, true);
+ ITypedStruct ts = structType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\ta : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\td : \t2\n" +
+ "\te : \t1\n" +
+ "\tf : \t1\n" +
+ "\tg : \t1\n" +
+ "\th : \t1.0\n" +
+ "\ti : \t1.0\n" +
+ "\tj : \t1\n" +
+ "\tk : \t1\n" +
+ "\tl : \t" + TEST_DATE + "\n" +
+ "\tm : \t[1, 1]\n" +
+ "\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
+ ".100000000000000088817841970012523233890533447265625]\n" +
+ "\to : \t{b=2.0, a=1.0}\n" +
+ "}");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/memory/TraitTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/memory/TraitTest.java b/repository/src/test/java/org/apache/atlas/repository/memory/TraitTest.java
new file mode 100755
index 0000000..583d43f
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/memory/TraitTest.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.BaseTest;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
+
+public class TraitTest extends BaseTest {
+
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ /*
+ * Type Hierarchy is:
+ * A(a,b,c,d)
+ * B(b) extends A
+ * C(c) extends A
+ * D(d) extends B,C
+ *
+ * - There are a total of 11 fields in an instance of D
+ * - an attribute that is hidden by a SubType can referenced by prefixing it with the
+ * complete Path.
+ * For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
+ * in B.
+ * So it is availabel by the name 'A.B.D.b'
+ *
+ * - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
+ * .b' attribute
+ * available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
+ * available as 'b'.
+ */
+ @Test
+ public void test1() throws MetadataException {
+ HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+ HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+ HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
+ HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+
+ defineTraits(A, B, C, D);
+
+ TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
+
+ Struct s1 = new Struct("D");
+ s1.set("d", 1);
+ s1.set("c", 1);
+ s1.set("b", true);
+ s1.set("a", 1);
+ s1.set("A.B.D.b", true);
+ s1.set("A.B.D.c", 2);
+ s1.set("A.B.D.d", 2);
+
+ s1.set("A.C.D.a", 3);
+ s1.set("A.C.D.b", false);
+ s1.set("A.C.D.c", 3);
+ s1.set("A.C.D.d", 3);
+
+
+ ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ /*
+ * cast to B and set the 'b' attribute on A.
+ */
+ TraitType BType = (TraitType) getTypeSystem().getDataType(TraitType.class, "B");
+ IStruct s2 = DType.castAs(ts, "B");
+ s2.set("A.B.b", false);
+
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \tfalse\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ /*
+ * cast again to A and set the 'b' attribute on A.
+ */
+ TraitType AType = (TraitType) getTypeSystem().getDataType(TraitType.class, "A");
+ IStruct s3 = BType.castAs(s2, "A");
+ s3.set("b", true);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+ }
+
+ @Test
+ public void testRandomOrder() throws MetadataException {
+ HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+ HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+ HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
+ HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+
+ defineTraits(B, D, A, C);
+
+ TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
+
+ Struct s1 = new Struct("D");
+ s1.set("d", 1);
+ s1.set("c", 1);
+ s1.set("b", true);
+ s1.set("a", 1);
+ s1.set("A.B.D.b", true);
+ s1.set("A.B.D.c", 2);
+ s1.set("A.B.D.d", 2);
+
+ s1.set("A.C.D.a", 3);
+ s1.set("A.C.D.b", false);
+ s1.set("A.C.D.c", 3);
+ s1.set("A.C.D.d", 3);
+
+
+ ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
+ Assert.assertEquals(ts.toString(), "{\n" +
+ "\td : \t1\n" +
+ "\tb : \ttrue\n" +
+ "\tc : \t1\n" +
+ "\ta : \t1\n" +
+ "\tA.B.D.b : \ttrue\n" +
+ "\tA.B.D.c : \t2\n" +
+ "\tA.B.D.d : \t2\n" +
+ "\tA.C.D.a : \t3\n" +
+ "\tA.C.D.b : \tfalse\n" +
+ "\tA.C.D.c : \t3\n" +
+ "\tA.C.D.d : \t3\n" +
+ "}");
+
+ }
+
+}
+
[24/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
deleted file mode 100755
index bf9dbb1..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
+++ /dev/null
@@ -1,676 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.google.common.collect.ImmutableList;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Compare;
-import com.tinkerpop.blueprints.GraphQuery;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.metadata.RepositoryMetadataModule;
-import org.apache.hadoop.metadata.TestUtils;
-import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.hadoop.metadata.repository.BaseTest;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.repository.EntityNotFoundException;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-import scala.actors.threadpool.Arrays;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * GraphBackedMetadataRepository test
- *
- * Guice loads the dependencies and injects the necessary objects
- *
- */
-@Guice(modules = RepositoryMetadataModule.class)
-public class GraphBackedMetadataRepositoryTest {
-
- private static final String ENTITY_TYPE = "Department";
- private static final String DATABASE_TYPE = "hive_database";
- private static final String DATABASE_NAME = "foo";
- private static final String TABLE_TYPE = "hive_table";
- private static final String TABLE_NAME = "bar";
- private static final String CLASSIFICATION = "classification";
- private static final String PII = "PII";
- private static final String SUPER_TYPE_NAME = "Base";
-
- @Inject
- private GraphProvider<TitanGraph> graphProvider;
-
- @Inject
- private GraphBackedMetadataRepository repositoryService;
-
- @Inject
- private GraphBackedDiscoveryService discoveryService;
-
- private TypeSystem typeSystem;
- private String guid;
-
- @BeforeClass
- public void setUp() throws Exception {
- typeSystem = TypeSystem.getInstance();
- typeSystem.reset();
-
- new GraphBackedSearchIndexer(graphProvider);
-
- TestUtils.defineDeptEmployeeTypes(typeSystem);
- createHiveTypes();
- }
-
-/*
- @AfterMethod
- public void tearDown() throws Exception {
- TestUtils.dumpGraph(graphProvider.get());
- }
-*/
-
- @Test
- public void testSubmitEntity() throws Exception {
- Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
- ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
-
- guid = repositoryService.createEntity(hrDept2);
- Assert.assertNotNull(guid);
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testGetEntityDefinitionForDepartment() throws Exception {
- ITypedReferenceableInstance entity = repositoryService.getEntityDefinition(guid);
- Assert.assertNotNull(entity);
- }
-
- @Test (expectedExceptions = RepositoryException.class)
- public void testGetEntityDefinitionNonExistent() throws Exception {
- repositoryService.getEntityDefinition("blah");
- Assert.fail();
- }
-
- @Test
- public void testGetEntityList() throws Exception {
- List<String> entityList = repositoryService.getEntityList(ENTITY_TYPE);
- System.out.println("entityList = " + entityList);
- Assert.assertNotNull(entityList);
- Assert.assertEquals(entityList.size(), 1); // one department
- }
-
- @Test
- public void testGetTypeAttributeName() throws Exception {
- Assert.assertEquals(
- repositoryService.getTypeAttributeName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
- }
-
- @Test (dependsOnMethods = "testSubmitEntity")
- public void testGetTraitLabel() throws Exception {
- Assert.assertEquals(repositoryService.getTraitLabel(
- typeSystem.getDataType(ClassType.class, TABLE_TYPE),
- CLASSIFICATION), TABLE_TYPE + "." + CLASSIFICATION);
- }
-
- @Test
- public void testCreateEntity() throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", DATABASE_NAME);
- databaseInstance.set("description", "foo database");
- databaseInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
-
- databaseInstance.set("namespace", "colo:cluster:hive:db");
- databaseInstance.set("cluster", "cluster-1");
- databaseInstance.set("colo", "colo-1");
- System.out.println("databaseInstance = " + databaseInstance);
-
- ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
- ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
- System.out.println("db = " + db);
-
- String dbGUID = repositoryService.createEntity(db);
- System.out.println("added db = " + dbGUID);
-
- Referenceable dbInstance = new Referenceable(
- dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
-
- ITypedReferenceableInstance table = createHiveTableInstance(dbInstance);
- String tableGUID = repositoryService.createEntity(table);
- System.out.println("added table = " + tableGUID);
- }
-
- @Test(dependsOnMethods = "testCreateEntity")
- public void testGetEntityDefinition() throws Exception {
- String guid = getGUID();
-
- ITypedReferenceableInstance table = repositoryService.getEntityDefinition(guid);
- Assert.assertEquals(table.getDate("created"), new Date(BaseTest.TEST_DATE_IN_LONG));
- System.out.println("*** table = " + table);
- }
-
- private String getGUID() {
- Vertex tableVertex = getTableEntityVertex();
-
- String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- if (guid == null) {
- Assert.fail();
- }
- return guid;
- }
-
- private Vertex getTableEntityVertex() {
- TitanGraph graph = graphProvider.get();
- GraphQuery query = graph.query()
- .has(Constants.ENTITY_TYPE_PROPERTY_KEY, Compare.EQUAL, TABLE_TYPE);
- Iterator<Vertex> results = query.vertices().iterator();
- // returning one since guid should be unique
- Vertex tableVertex = results.hasNext() ? results.next() : null;
- if (tableVertex == null) {
- Assert.fail();
- }
-
- return tableVertex;
- }
-
- @Test (dependsOnMethods = "testCreateEntity")
- public void testGetTraitNames() throws Exception {
- final List<String> traitNames = repositoryService.getTraitNames(getGUID());
- Assert.assertEquals(traitNames.size(), 1);
- Assert.assertEquals(traitNames, Arrays.asList(new String[]{CLASSIFICATION}));
- }
-
- @Test
- public void testGetTraitNamesForEmptyTraits() throws Exception {
- final List<String> traitNames = repositoryService.getTraitNames(guid);
- Assert.assertEquals(traitNames.size(), 0);
- }
-
- @Test (expectedExceptions = EntityNotFoundException.class)
- public void testGetTraitNamesForBadEntity() throws Exception {
- repositoryService.getTraitNames(UUID.randomUUID().toString());
- Assert.fail();
- }
-
- @Test (dependsOnMethods = "testGetTraitNames")
- public void testAddTrait() throws Exception {
- final String aGUID = getGUID();
-
- List<String> traitNames = repositoryService.getTraitNames(aGUID);
- System.out.println("traitNames = " + traitNames);
- Assert.assertEquals(traitNames.size(), 1);
- Assert.assertTrue(traitNames.contains(CLASSIFICATION));
- Assert.assertFalse(traitNames.contains(PII));
-
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(PII, ImmutableList.<String>of());
- TraitType traitType = typeSystem.defineTraitType(piiTrait);
- ITypedStruct traitInstance = traitType.createInstance();
-
- repositoryService.addTrait(aGUID, traitInstance);
-
- // refresh trait names
- traitNames = repositoryService.getTraitNames(aGUID);
- Assert.assertEquals(traitNames.size(), 2);
- Assert.assertTrue(traitNames.contains(PII));
- Assert.assertTrue(traitNames.contains(CLASSIFICATION));
- }
-
- @Test (dependsOnMethods = "testAddTrait")
- public void testAddTraitWithAttribute() throws Exception {
- final String aGUID = getGUID();
- final String traitName = "P_I_I";
-
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
- TraitType traitType = typeSystem.defineTraitType(piiTrait);
- ITypedStruct traitInstance = traitType.createInstance();
- traitInstance.set("type", "SSN");
-
- repositoryService.addTrait(aGUID, traitInstance);
-
- TestUtils.dumpGraph(graphProvider.get());
-
- // refresh trait names
- List<String> traitNames = repositoryService.getTraitNames(aGUID);
- Assert.assertEquals(traitNames.size(), 3);
- Assert.assertTrue(traitNames.contains(traitName));
-
- ITypedReferenceableInstance instance = repositoryService.getEntityDefinition(aGUID);
- IStruct traitInstanceRef = instance.getTrait(traitName);
- String type = (String) traitInstanceRef.get("type");
- Assert.assertEquals(type, "SSN");
- }
-
- @Test (expectedExceptions = NullPointerException.class)
- public void testAddTraitWithNullInstance() throws Exception {
- repositoryService.addTrait(getGUID(), null);
- Assert.fail();
- }
-
- @Test (dependsOnMethods = "testAddTrait", expectedExceptions = RepositoryException.class)
- public void testAddTraitForBadEntity() throws Exception {
- TraitType traitType = typeSystem.getDataType(TraitType.class, PII);
- ITypedStruct traitInstance = traitType.createInstance();
-
- repositoryService.addTrait(UUID.randomUUID().toString(), traitInstance);
- Assert.fail();
- }
-
- @Test (dependsOnMethods = "testAddTrait")
- public void testDeleteTrait() throws Exception {
- final String aGUID = getGUID();
-
- List<String> traitNames = repositoryService.getTraitNames(aGUID);
- Assert.assertEquals(traitNames.size(), 3);
- Assert.assertTrue(traitNames.contains(PII));
- Assert.assertTrue(traitNames.contains(CLASSIFICATION));
- Assert.assertTrue(traitNames.contains("P_I_I"));
-
- repositoryService.deleteTrait(aGUID, PII);
-
- // refresh trait names
- traitNames = repositoryService.getTraitNames(aGUID);
- Assert.assertEquals(traitNames.size(), 2);
- Assert.assertTrue(traitNames.contains(CLASSIFICATION));
- Assert.assertFalse(traitNames.contains(PII));
- }
-
- @Test (expectedExceptions = RepositoryException.class)
- public void testDeleteTraitForNonExistentEntity() throws Exception {
- repositoryService.deleteTrait(UUID.randomUUID().toString(), PII);
- Assert.fail();
- }
-
- @Test (expectedExceptions = RepositoryException.class)
- public void testDeleteTraitForNonExistentTrait() throws Exception {
- final String aGUID = getGUID();
- repositoryService.deleteTrait(aGUID, "PCI");
- Assert.fail();
- }
-
- @Test (dependsOnMethods = "testCreateEntity")
- public void testGetIdFromVertex() throws Exception {
- Vertex tableVertex = getTableEntityVertex();
-
- String guid = tableVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- if (guid == null) {
- Assert.fail();
- }
-
- Id expected = new Id(guid,
- tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TABLE_TYPE);
- Assert.assertEquals(repositoryService.getIdFromVertex(TABLE_TYPE, tableVertex), expected);
- }
-
- @Test (dependsOnMethods = "testCreateEntity")
- public void testGetTypeName() throws Exception {
- Vertex tableVertex = getTableEntityVertex();
- Assert.assertEquals(repositoryService.getTypeName(tableVertex), TABLE_TYPE);
- }
-
- @Test(dependsOnMethods = "testCreateEntity")
- public void testSearchByDSLQuery() throws Exception {
- String dslQuery = "hive_database as PII";
- System.out.println("Executing dslQuery = " + dslQuery);
- String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
-
- JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
- System.out.println("results = " + results);
-
- Object query = results.get("query");
- Assert.assertNotNull(query);
-
- JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
- String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertTrue(rows.length() > 0);
-
- for (int index = 0; index < rows.length(); index++) {
- JSONObject row = rows.getJSONObject(index);
- String type = row.getString("$typeName$");
- Assert.assertEquals(type, "hive_database");
-
- String name = row.getString("name");
- Assert.assertEquals(name, DATABASE_NAME);
- }
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testSearchByDSLWithInheritance() throws Exception {
- String dslQuery = "Person where name = 'Jane'";
- System.out.println("Executing dslQuery = " + dslQuery);
- String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
-
- JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
- System.out.println("results = " + results);
-
- Object query = results.get("query");
- Assert.assertNotNull(query);
-
- JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
- String typeName = dataType.getString("typeName");
- Assert.assertEquals(typeName, "Person");
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertEquals(rows.length(), 1);
-
- JSONObject row = rows.getJSONObject(0);
- Assert.assertEquals(row.getString("$typeName$"), "Manager");
- Assert.assertEquals(row.getString("name"), "Jane");
- }
-
- @Test(dependsOnMethods = "testCreateEntity")
- public void testBug37860() throws Exception {
- String dslQuery =
- "hive_table as t where name = 'bar' " +
- "database where name = 'foo' and description = 'foo database' select t";
- System.out.println("Executing dslQuery = " + dslQuery);
- String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
-
- JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
- System.out.println("results = " + results);
-
- Object query = results.get("query");
- Assert.assertNotNull(query);
-
- JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertEquals(rows.length(), 1);
-
- }
-
- /**
- * Full text search requires GraphBackedSearchIndexer, and GraphBackedSearchIndexer can't be enabled in
- * GraphBackedDiscoveryServiceTest because of its test data. So, test for full text search is in
- * GraphBackedMetadataRepositoryTest:(
- */
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testFullTextSearch() throws Exception {
- //todo fix this
- //Weird: with lucene, the test passes without sleep
- //but with elasticsearch, doesn't work without sleep. why??
- long sleepInterval = 1000;
-
- //person in hr department whose name is john
- Thread.sleep(sleepInterval);
- String response = discoveryService.searchByFullText("john");
- Assert.assertNotNull(response);
- JSONArray results = new JSONArray(response);
- Assert.assertEquals(results.length(), 1);
- JSONObject row = (JSONObject) results.get(0);
- Assert.assertEquals(row.get("typeName"), "Person");
-
- //person in hr department who lives in santa clara
- response = discoveryService.searchByFullText("Jane AND santa AND clara");
- Assert.assertNotNull(response);
- results = new JSONArray(response);
- Assert.assertEquals(results.length(), 1);
- row = (JSONObject) results.get(0);
- Assert.assertEquals(row.get("typeName"), "Manager");
-
- //search for person in hr department whose name starts is john/jahn
- response = discoveryService.searchByFullText("hr AND (john OR jahn)");
- Assert.assertNotNull(response);
- results = new JSONArray(response);
- Assert.assertEquals(results.length(), 1);
- row = (JSONObject) results.get(0);
- Assert.assertEquals(row.get("typeName"), "Person");
- }
-
- private void createHiveTypes() throws Exception {
- HierarchicalTypeDefinition<ClassType> superTypeDefinition =
- TypesUtil.createClassTypeDef(SUPER_TYPE_NAME,
- ImmutableList.<String>of(),
- TypesUtil.createOptionalAttrDef("namespace", DataTypes.STRING_TYPE),
- TypesUtil.createOptionalAttrDef("cluster", DataTypes.STRING_TYPE),
- TypesUtil.createOptionalAttrDef("colo", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
- TypesUtil.createClassTypeDef(DATABASE_TYPE,
- ImmutableList.of(SUPER_TYPE_NAME),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
-
- StructTypeDefinition structTypeDefinition =
- new StructTypeDefinition("serdeType",
- new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
- });
-
- EnumValue values[] = {
- new EnumValue("MANAGED", 1),
- new EnumValue("EXTERNAL", 2),
- };
-
- EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
- typeSystem.defineEnumType(enumTypeDefinition);
-
- HierarchicalTypeDefinition<ClassType> columnsDefinition =
- TypesUtil.createClassTypeDef("column_type",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
-
- StructTypeDefinition partitionDefinition =
- new StructTypeDefinition("partition_type",
- new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- });
-
- HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
- TypesUtil.createClassTypeDef(TABLE_TYPE,
- ImmutableList.of(SUPER_TYPE_NAME),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
- TypesUtil.createOptionalAttrDef("created", DataTypes.DATE_TYPE),
- // enum
- new AttributeDefinition("tableType", "tableType",
- Multiplicity.REQUIRED, false, null),
- // array of strings
- new AttributeDefinition("columnNames",
- String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
- Multiplicity.COLLECTION, false, null),
- // array of classes
- new AttributeDefinition("columns",
- String.format("array<%s>", "column_type"),
- Multiplicity.COLLECTION, true, null),
- // array of structs
- new AttributeDefinition("partitions",
- String.format("array<%s>", "partition_type"),
- Multiplicity.COLLECTION, true, null),
- // map of primitives
- new AttributeDefinition("parametersMap",
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
- DataTypes.STRING_TYPE.getName()),
- Multiplicity.COLLECTION, true, null),
- // map of classes - todo - enable this
-// new AttributeDefinition("columnsMap",
-// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
-// "column_type"),
-// Multiplicity.COLLECTION, true, null),
- // map of structs todo - enable this
-// new AttributeDefinition("partitionsMap",
-// DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
-// "partition_type"),
-// Multiplicity.COLLECTION, true, null),
- // struct reference
- new AttributeDefinition("serde1",
- "serdeType", Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("serde2",
- "serdeType", Multiplicity.REQUIRED, false, null),
- // class reference
- new AttributeDefinition("database",
- DATABASE_TYPE, Multiplicity.REQUIRED, true, null)
- );
-
- HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
- TypesUtil.createTraitTypeDef(CLASSIFICATION,
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition =
- TypesUtil.createTraitTypeDef("fetl" + CLASSIFICATION,
- ImmutableList.of(CLASSIFICATION),
- TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
-
- typeSystem.defineTypes(
- ImmutableList.of(structTypeDefinition, partitionDefinition),
- ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition),
- ImmutableList.of(superTypeDefinition, databaseTypeDefinition,
- columnsDefinition, tableTypeDefinition));
- }
-
- private ITypedReferenceableInstance createHiveTableInstance(
- Referenceable databaseInstance) throws Exception {
- Referenceable tableInstance = new Referenceable(TABLE_TYPE, CLASSIFICATION);
- tableInstance.set("name", TABLE_NAME);
- tableInstance.set("description", "bar table");
- tableInstance.set("type", "managed");
- tableInstance.set("created", new Date(BaseTest.TEST_DATE_IN_LONG));
- tableInstance.set("tableType", 1); // enum
-
- // super type
- tableInstance.set("namespace", "colo:cluster:hive:db:table");
- tableInstance.set("cluster", "cluster-1");
- tableInstance.set("colo", "colo-1");
-
- // refer to an existing class
- tableInstance.set("database", databaseInstance);
-
- ArrayList<String> columnNames = new ArrayList<>();
- columnNames.add("first_name");
- columnNames.add("last_name");
- tableInstance.set("columnNames", columnNames);
-
- Struct traitInstance = (Struct) tableInstance.getTrait(CLASSIFICATION);
- traitInstance.set("tag", "foundation_etl");
-
- Struct serde1Instance = new Struct("serdeType");
- serde1Instance.set("name", "serde1");
- serde1Instance.set("serde", "serde1");
- tableInstance.set("serde1", serde1Instance);
-
- Struct serde2Instance = new Struct("serdeType");
- serde2Instance.set("name", "serde2");
- serde2Instance.set("serde", "serde2");
- tableInstance.set("serde2", serde2Instance);
-
- // HashMap<String, Referenceable> columnsMap = new HashMap<>();
- ArrayList<Referenceable> columns = new ArrayList<>();
- for (int index = 0; index < 5; index++) {
- Referenceable columnInstance = new Referenceable("column_type");
- final String name = "column_" + index;
- columnInstance.set("name", name);
- columnInstance.set("type", "string");
-
- columns.add(columnInstance);
- // columnsMap.put(name, columnInstance);
- }
- tableInstance.set("columns", columns);
- // tableInstance.set("columnsMap", columnsMap);
-
-// HashMap<String, Struct> partitionsMap = new HashMap<>();
- ArrayList<Struct> partitions = new ArrayList<>();
- for (int index = 0; index < 5; index++) {
- Struct partitionInstance = new Struct("partition_type");
- final String name = "partition_" + index;
- partitionInstance.set("name", name);
-
- partitions.add(partitionInstance);
-// partitionsMap.put(name, partitionInstance);
- }
- tableInstance.set("partitions", partitions);
-// tableInstance.set("partitionsMap", partitionsMap);
-
- HashMap<String, String> parametersMap = new HashMap<>();
- parametersMap.put("foo", "bar");
- parametersMap.put("bar", "baz");
- parametersMap.put("some", "thing");
- tableInstance.set("parametersMap", parametersMap);
-
- ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
- return tableType.convert(tableInstance, Multiplicity.REQUIRED);
- }
-
- private String random() {
- return RandomStringUtils.random(10);
- }
-
- @Test
- public void testUTFValues() throws Exception {
- Referenceable hrDept = new Referenceable("Department");
- Referenceable john = new Referenceable("Person");
- john.set("name", random());
- john.set("department", hrDept);
-
- hrDept.set("name", random());
- hrDept.set("employees", ImmutableList.of(john));
-
- ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
-
- guid = repositoryService.createEntity(hrDept2);
- Assert.assertNotNull(guid);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperScaleTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperScaleTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperScaleTest.java
deleted file mode 100755
index fd1990e..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphRepoMapperScaleTest.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.graph;
-
-import com.google.common.collect.ImmutableList;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanIndexQuery;
-import com.tinkerpop.blueprints.Compare;
-import com.tinkerpop.blueprints.GraphQuery;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.hadoop.metadata.RepositoryMetadataModule;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumType;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Map;
-
-@Test
-@Guice(modules = RepositoryMetadataModule.class)
-public class GraphRepoMapperScaleTest {
-
- private static final String DATABASE_TYPE = "hive_database_type";
- private static final String DATABASE_NAME = "foo";
- private static final String TABLE_TYPE = "hive_table_type";
- private static final String TABLE_NAME = "bar";
-
- @Inject
- private GraphProvider<TitanGraph> graphProvider;
- @Inject
- private GraphBackedMetadataRepository repositoryService;
-
- private GraphBackedSearchIndexer searchIndexer;
- private TypeSystem typeSystem;
- private String dbGUID;
-
- @BeforeClass
- public void setUp() throws Exception {
- searchIndexer = new GraphBackedSearchIndexer(graphProvider);
-
- typeSystem = TypeSystem.getInstance();
-
- createHiveTypes();
- }
-
- @Test
- public void testSubmitEntity() throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", DATABASE_NAME);
- databaseInstance.set("description", "foo database");
- // System.out.println("databaseInstance = " + databaseInstance);
-
- ClassType dbType = typeSystem.getDataType(ClassType.class, DATABASE_TYPE);
- ITypedReferenceableInstance db = dbType.convert(databaseInstance, Multiplicity.REQUIRED);
-
- dbGUID = repositoryService.createEntity(db);
-
- Referenceable dbInstance = new Referenceable(
- dbGUID, DATABASE_TYPE, databaseInstance.getValuesMap());
-
- for (int index = 0; index < 1000; index++) {
- ITypedReferenceableInstance table = createHiveTableInstance(dbInstance, index);
- repositoryService.createEntity(table);
- }
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testSearchIndex() throws Exception {
- searchWithOutIndex(Constants.GUID_PROPERTY_KEY, dbGUID);
- searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, "hive_column_type");
- searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, TABLE_TYPE);
-
- searchWithOutIndex("hive_table_type.name", "bar-999");
- searchWithIndex("hive_table_type.name", "bar-999");
-
- for (int index = 500; index < 600; index++) {
- searchWithIndex("hive_table_type.name", "bar-" + index);
- }
- }
-
- private void searchWithOutIndex(String key, String value) {
- TitanGraph graph = graphProvider.get();
- long start = System.currentTimeMillis();
- int count = 0;
- try {
- GraphQuery query = graph.query()
- .has(key, Compare.EQUAL, value);
- for (Vertex ignored : query.vertices()) {
- count++;
- }
- } finally {
- System.out.println("Search on [" + key + "=" + value + "] returned results: " + count
- + ", took " + (System.currentTimeMillis() - start) + " ms");
- }
- }
-
- private void searchWithIndex(String key, String value) {
- TitanGraph graph = graphProvider.get();
- long start = System.currentTimeMillis();
- int count = 0;
- try {
- String queryString = "v.\"" + key + "\":(" + value + ")";
- TitanIndexQuery query = graph.indexQuery(Constants.VERTEX_INDEX, queryString);
- for (TitanIndexQuery.Result<Vertex> ignored : query.vertices()) {
- count++;
- }
- } finally {
- System.out.println("Search on [" + key + "=" + value + "] returned results: " + count
- + ", took " + (System.currentTimeMillis() - start) + " ms");
- }
- }
-
- private void createHiveTypes() throws Exception {
- HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
- TypesUtil.createClassTypeDef(DATABASE_TYPE,
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
- StructTypeDefinition structTypeDefinition =
- new StructTypeDefinition("hive_serde_type",
- new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
- });
-
- EnumValue values[] = {
- new EnumValue("MANAGED", 1),
- new EnumValue("EXTERNAL", 2),
- };
-
- EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("table_type", values);
- EnumType enumType = typeSystem.defineEnumType(enumTypeDefinition);
- searchIndexer.onAdd("table_type", enumType);
-
- HierarchicalTypeDefinition<ClassType> columnsDefinition =
- TypesUtil.createClassTypeDef("hive_column_type",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
-
- StructTypeDefinition partitionDefinition =
- new StructTypeDefinition("hive_partition_type",
- new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- });
-
- HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
- TypesUtil.createClassTypeDef(TABLE_TYPE,
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
- // enum
- new AttributeDefinition("tableType", "table_type",
- Multiplicity.REQUIRED, false, null),
- // array of strings
- new AttributeDefinition("columnNames",
- String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
- Multiplicity.COLLECTION, false, null),
- // array of classes
- new AttributeDefinition("columns",
- String.format("array<%s>", "hive_column_type"),
- Multiplicity.COLLECTION, true, null),
- // array of structs
- new AttributeDefinition("partitions",
- String.format("array<%s>", "hive_partition_type"),
- Multiplicity.COLLECTION, true, null),
- // struct reference
- new AttributeDefinition("serde1",
- "hive_serde_type", Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("serde2",
- "hive_serde_type", Multiplicity.REQUIRED, false, null),
- // class reference
- new AttributeDefinition("database",
- DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
-
- HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
- TypesUtil.createTraitTypeDef("pii_type", ImmutableList.<String>of());
-
- Map<String, IDataType> types = typeSystem.defineTypes(
- ImmutableList.of(structTypeDefinition, partitionDefinition),
- ImmutableList.of(classificationTypeDefinition),
- ImmutableList.of(databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
-
- for (Map.Entry<String, IDataType> entry : types.entrySet()) {
- searchIndexer.onAdd(entry.getKey(), entry.getValue());
- }
- }
-
- private ITypedReferenceableInstance createHiveTableInstance(
- Referenceable databaseInstance, int uberIndex) throws Exception {
-
- Referenceable tableInstance = new Referenceable(TABLE_TYPE, "pii_type");
- tableInstance.set("name", TABLE_NAME + "-" + uberIndex);
- tableInstance.set("description", "bar table" + "-" + uberIndex);
- tableInstance.set("type", "managed");
- tableInstance.set("tableType", 1); // enum
-
- // refer to an existing class
- tableInstance.set("database", databaseInstance);
-
- ArrayList<String> columnNames = new ArrayList<>();
- columnNames.add("first_name" + "-" + uberIndex);
- columnNames.add("last_name" + "-" + uberIndex);
- tableInstance.set("columnNames", columnNames);
-
- Struct serde1Instance = new Struct("hive_serde_type");
- serde1Instance.set("name", "serde1" + "-" + uberIndex);
- serde1Instance.set("serde", "serde1" + "-" + uberIndex);
- tableInstance.set("serde1", serde1Instance);
-
- Struct serde2Instance = new Struct("hive_serde_type");
- serde2Instance.set("name", "serde2" + "-" + uberIndex);
- serde2Instance.set("serde", "serde2" + "-" + uberIndex);
- tableInstance.set("serde2", serde2Instance);
-
- ArrayList<Referenceable> columns = new ArrayList<>();
- for (int index = 0; index < 5; index++) {
- Referenceable columnInstance = new Referenceable("hive_column_type");
- columnInstance.set("name", "column_" + "-" + uberIndex + "-" + index);
- columnInstance.set("type", "string");
- columns.add(columnInstance);
- }
- tableInstance.set("columns", columns);
-
- ArrayList<Struct> partitions = new ArrayList<>();
- for (int index = 0; index < 5; index++) {
- Struct partitionInstance = new Struct("hive_partition_type");
- partitionInstance.set("name", "partition_" + "-" + uberIndex + "-" + index);
- partitions.add(partitionInstance);
- }
- tableInstance.set("partitions", partitions);
-
- ClassType tableType = typeSystem.getDataType(ClassType.class, TABLE_TYPE);
- return tableType.convert(tableInstance, Multiplicity.REQUIRED);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/ClassTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/ClassTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/ClassTest.java
deleted file mode 100755
index e101a50..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/ClassTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.BaseTest;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ClassTest extends BaseTest {
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- @Test
- public void test1() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
-
- defineDeptEmployeeTypes(ts);
- Referenceable hrDept = createDeptEg1(ts);
- ClassType deptType = ts.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
-
-
- Assert.assertEquals(hrDept2.toString(), "{\n" +
- "\tid : (type: Department, id: <unassigned>)\n" +
- "\tname : \thr\n" +
- "\temployees : \t[{\n" +
- "\tid : (type: Person, id: <unassigned>)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : (type: Manager, id: <unassigned>)\n" +
- "}, {\n" +
- "\tid : (type: Manager, id: <unassigned>)\n" +
- "\tsubordinates : \t[{\n" +
- "\tid : (type: Person, id: <unassigned>)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : (type: Manager, id: <unassigned>)\n" +
- "}]\n" +
- "\tname : \tJane\n" +
- "\tdepartment : (type: Department, id: <unassigned>)\n" +
- "\tmanager : <null>\n" +
- "\n" +
- "\tSecurityClearance : \t{\n" +
- "\t\tlevel : \t\t1\n" +
- "\t}}]\n" +
- "}");
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java
deleted file mode 100755
index b279c0b..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/EnumTest.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.json.Serialization$;
-import org.apache.hadoop.metadata.repository.BaseTest;
-import org.apache.hadoop.metadata.typesystem.IReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumType;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-
-public class EnumTest extends BaseTest {
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- void defineEnums(TypeSystem ts) throws MetadataException {
- ts.defineEnumType("HiveObjectType",
- new EnumValue("GLOBAL", 1),
- new EnumValue("DATABASE", 2),
- new EnumValue("TABLE", 3),
- new EnumValue("PARTITION", 4),
- new EnumValue("COLUMN", 5));
-
- ts.defineEnumType("PrincipalType",
- new EnumValue("USER", 1),
- new EnumValue("ROLE", 2),
- new EnumValue("GROUP", 3));
-
- ts.defineEnumType("TxnState",
- new EnumValue("COMMITTED", 1),
- new EnumValue("ABORTED", 2),
- new EnumValue("OPEN", 3));
-
- ts.defineEnumType("LockLevel",
- new EnumValue("DB", 1),
- new EnumValue("TABLE", 2),
- new EnumValue("PARTITION", 3));
-
- }
-
- protected void fillStruct(Struct s) throws MetadataException {
- s.set("a", 1);
- s.set("b", true);
- s.set("c", (byte) 1);
- s.set("d", (short) 2);
- s.set("e", 1);
- s.set("f", 1);
- s.set("g", 1L);
- s.set("h", 1.0f);
- s.set("i", 1.0);
- s.set("j", BigInteger.valueOf(1L));
- s.set("k", new BigDecimal(1));
- s.set("l", new Date(1418265358440L));
- s.set("m", Lists.asList(1, new Integer[]{1}));
- s.set("n",
- Lists.asList(BigDecimal.valueOf(1.1), new BigDecimal[]{BigDecimal.valueOf(1.1)}));
- Map<String, Double> hm = Maps.newHashMap();
- hm.put("a", 1.0);
- hm.put("b", 2.0);
- s.set("o", hm);
- s.set("enum1", "GLOBAL");
- s.set("enum2", 1);
- s.set("enum3", "COMMITTED");
- s.set("enum4", 3);
- }
-
- protected Struct createStructWithEnum(String typeName) throws MetadataException {
- Struct s = new Struct(typeName);
- fillStruct(s);
- return s;
- }
-
- protected Referenceable createInstanceWithEnum(String typeName) throws MetadataException {
- Referenceable r = new Referenceable(typeName);
- fillStruct(r);
- return r;
- }
-
- protected ClassType defineClassTypeWithEnum(TypeSystem ts) throws MetadataException {
- return ts.defineClassType(TypesUtil.createClassTypeDef("t4",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
- TypesUtil.createOptionalAttrDef("enum1",
- ts.getDataType(EnumType.class, "HiveObjectType")),
- TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
- TypesUtil.createOptionalAttrDef("enum2",
- ts.getDataType(EnumType.class, "PrincipalType")),
- TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
- TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
- TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
- TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
- TypesUtil
- .createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
- TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
- TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- TypesUtil.createOptionalAttrDef("o",
- ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- TypesUtil.createOptionalAttrDef("enum4",
- ts.getDataType(EnumType.class, "LockLevel"))));
- }
-
- @Test
- public void testStruct() throws MetadataException {
- TypeSystem ts = getTypeSystem();
- defineEnums(ts);
- StructType structType = ts.defineStructType("t3",
- true,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
- TypesUtil.createOptionalAttrDef("enum1",
- ts.getDataType(EnumType.class, "HiveObjectType")),
- TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
- TypesUtil.createOptionalAttrDef("enum2",
- ts.getDataType(EnumType.class, "PrincipalType")),
- TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
- TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
- TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
- TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
- TypesUtil
- .createOptionalAttrDef("enum3", ts.getDataType(EnumType.class, "TxnState")),
-
- TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
- TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- TypesUtil.createOptionalAttrDef("o",
- ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- TypesUtil.createOptionalAttrDef("enum4",
- ts.getDataType(EnumType.class, "LockLevel")));
-
- Struct s = createStructWithEnum("t3");
- ITypedStruct typedS = structType.convert(s, Multiplicity.REQUIRED);
- Assert.assertEquals(typedS.toString(), "{\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t2\n" +
- "\tenum1 : \tGLOBAL\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\tenum2 : \tUSER\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tenum3 : \tCOMMITTED\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.1, 1.1]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "\tenum4 : \tPARTITION\n" +
- "}");
- }
-
- @Test
- public void testClass() throws MetadataException {
- TypeSystem ts = getTypeSystem();
- defineEnums(ts);
- ClassType clsType = defineClassTypeWithEnum(ts);
-
- IReferenceableInstance r = createInstanceWithEnum("t4");
- ITypedReferenceableInstance typedR = clsType.convert(r, Multiplicity.REQUIRED);
- Assert.assertEquals(typedR.toString(), "{\n" +
- "\tid : (type: t4, id: <unassigned>)\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t2\n" +
- "\tenum1 : \tGLOBAL\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\tenum2 : \tUSER\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tenum3 : \tCOMMITTED\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.1, 1.1]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "\tenum4 : \tPARTITION\n" +
- "}");
- }
-
- @Test
- public void testStorage() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
- defineEnums(ts);
- ClassType clsType = defineClassTypeWithEnum(ts);
-
- getRepository().defineTypes(ImmutableList.of((HierarchicalType) clsType));
-
- IReferenceableInstance r = createInstanceWithEnum("t4");
- IReferenceableInstance r1 = getRepository().create(r);
-
- ITypedReferenceableInstance r2 = getRepository().get(r1.getId());
- Assert.assertEquals(r2.toString(), "{\n" +
- "\tid : (type: t4, id: 1)\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t0\n" +
- "\tenum1 : \tGLOBAL\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\tenum2 : \tUSER\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tenum3 : \tCOMMITTED\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.1, 1.1]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "\tenum4 : \tPARTITION\n" +
- "}");
- }
-
- @Test
- public void testJson() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
- defineEnums(ts);
- ClassType clsType = defineClassTypeWithEnum(ts);
-
- getRepository().defineTypes(ImmutableList.of((HierarchicalType) clsType));
-
- IReferenceableInstance r = createInstanceWithEnum("t4");
- IReferenceableInstance r1 = getRepository().create(r);
-
- ITypedReferenceableInstance r2 = getRepository().get(r1.getId());
- String jsonStr = Serialization$.MODULE$.toJson(r2);
-
- IReferenceableInstance r3 = Serialization$.MODULE$.fromJson(jsonStr);
- Assert.assertEquals(r3.toString(), "{\n" +
- "\tid : (type: t4, id: 1)\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t0\n" +
- "\tenum1 : \tGLOBAL\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\tenum2 : \tUSER\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tenum3 : \tCOMMITTED\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
- ".100000000000000088817841970012523233890533447265625]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "\tenum4 : \tPARTITION\n" +
- "}");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/InstanceE2ETest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/InstanceE2ETest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/InstanceE2ETest.java
deleted file mode 100755
index 34afcbd..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/InstanceE2ETest.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.*;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization$;
-import org.apache.hadoop.metadata.typesystem.json.Serialization$;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization$;
-import org.apache.hadoop.metadata.repository.BaseTest;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class InstanceE2ETest extends BaseTest {
-
- protected List<HierarchicalTypeDefinition> createHiveTypes(TypeSystem typeSystem)
- throws MetadataException {
- ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
-
- HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
- TypesUtil.createClassTypeDef("hive_database",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
- typeDefinitions.add(databaseTypeDefinition);
-
- HierarchicalTypeDefinition<ClassType> tableTypeDefinition = TypesUtil.createClassTypeDef(
- "hive_table",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
- new AttributeDefinition("hive_database",
- "hive_database", Multiplicity.REQUIRED, false, "hive_database"));
- typeDefinitions.add(tableTypeDefinition);
-
- HierarchicalTypeDefinition<TraitType> fetlTypeDefinition = TypesUtil.createTraitTypeDef(
- "hive_fetl",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
- typeDefinitions.add(fetlTypeDefinition);
-
- typeSystem.defineTypes(
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(fetlTypeDefinition),
- ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
-
- return typeDefinitions;
- }
-
- protected Referenceable createHiveTableReferenceable()
- throws MetadataException {
- Referenceable databaseInstance = new Referenceable("hive_database");
- databaseInstance.set("name", "hive_database");
- databaseInstance.set("description", "foo database");
-
- Referenceable tableInstance = new Referenceable("hive_table", "hive_fetl");
- tableInstance.set("name", "t1");
- tableInstance.set("description", "bar table");
- tableInstance.set("type", "managed");
- tableInstance.set("hive_database", databaseInstance);
-
- Struct traitInstance = (Struct) tableInstance.getTrait("hive_fetl");
- traitInstance.set("level", 1);
-
- tableInstance.set("hive_fetl", traitInstance);
-
- return tableInstance;
- }
-
- protected ITypedReferenceableInstance createHiveTableInstance(TypeSystem typeSystem)
- throws MetadataException {
- ClassType tableType = typeSystem.getDataType(ClassType.class, "hive_table");
- return tableType.convert(createHiveTableReferenceable(), Multiplicity.REQUIRED);
- }
-
- @Test
- public void testType() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
-
- createHiveTypes(ts);
-
- String jsonStr = TypesSerialization$.MODULE$
- .toJson(ts, ImmutableList.of("hive_database", "hive_table"));
- System.out.println(jsonStr);
-
- TypesDef typesDef1 = TypesSerialization$.MODULE$.fromJson(jsonStr);
- System.out.println(typesDef1);
-
- ts.reset();
- ts.defineTypes(typesDef1);
- jsonStr = TypesSerialization$.MODULE$
- .toJson(ts, ImmutableList.of("hive_database", "hive_table"));
- System.out.println(jsonStr);
-
- }
-
- @Test
- public void testInstance() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
-
- createHiveTypes(ts);
-
- ITypedReferenceableInstance i = createHiveTableInstance(getTypeSystem());
-
- String jsonStr = Serialization$.MODULE$.toJson(i);
- System.out.println(jsonStr);
-
- i = Serialization$.MODULE$.fromJson(jsonStr);
- System.out.println(i);
- }
-
- @Test
- public void testInstanceSerialization() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
-
- createHiveTypes(ts);
-
- Referenceable r = createHiveTableReferenceable();
- String jsonStr = InstanceSerialization$.MODULE$.toJson(r, true);
- Referenceable r1 = InstanceSerialization$.MODULE$.fromJsonReferenceable(jsonStr, true);
- ClassType tableType = ts.getDataType(ClassType.class, "hive_table");
-
- ITypedReferenceableInstance i = tableType.convert(r1, Multiplicity.REQUIRED);
-
- jsonStr = Serialization$.MODULE$.toJson(i);
- System.out.println(jsonStr);
-
- i = Serialization$.MODULE$.fromJson(jsonStr);
- System.out.println(i);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StorageTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StorageTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StorageTest.java
deleted file mode 100755
index d967394..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StorageTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.BaseTest;
-import org.apache.hadoop.metadata.repository.RepositoryException;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class StorageTest extends BaseTest {
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- @Test
- public void test1() throws MetadataException {
-
- TypeSystem ts = getTypeSystem();
-
- defineDeptEmployeeTypes(ts);
-
- Referenceable hrDept = createDeptEg1(ts);
- ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
- ITypedReferenceableInstance hrDept3 = getRepository().get(hrDept2.getId());
- Assert.assertEquals(hrDept3.toString(), "{\n" +
- "\tid : (type: Department, id: 1)\n" +
- "\tname : \thr\n" +
- "\temployees : \t[{\n" +
- "\tid : (type: Person, id: 2)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: 1)\n" +
- "\tmanager : (type: Manager, id: 3)\n" +
- "}, {\n" +
- "\tid : (type: Manager, id: 3)\n" +
- "\tsubordinates : \t[(type: Person, id: 2)]\n" +
- "\tname : \tJane\n" +
- "\tdepartment : (type: Department, id: 1)\n" +
- "\tmanager : <null>\n" +
- "\n" +
- "\tSecurityClearance : \t{\n" +
- "\t\tlevel : \t\t1\n" +
- "\t}}]\n" +
- "}");
- }
-
- @Test
- public void testGetPerson() throws MetadataException {
- TypeSystem ts = getTypeSystem();
- defineDeptEmployeeTypes(ts);
-
- Referenceable hrDept = createDeptEg1(ts);
- ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
-
- Id e1Id = new Id(2, 0, "Person");
- ITypedReferenceableInstance e1 = getRepository().get(e1Id);
- Assert.assertEquals(e1.toString(), "{\n" +
- "\tid : (type: Person, id: 2)\n" +
- "\tname : \tJohn\n" +
- "\tdepartment : (type: Department, id: 1)\n" +
- "\tmanager : (type: Manager, id: 3)\n" +
- "}");
- }
-
- @Test
- public void testInvalidTypeName() throws MetadataException {
- TypeSystem ts = getTypeSystem();
- defineDeptEmployeeTypes(ts);
-
- Referenceable hrDept = createDeptEg1(ts);
- ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
-
- Id e1Id = new Id(3, 0, "Person");
- try {
- ITypedReferenceableInstance e1 = getRepository().get(e1Id);
- } catch (RepositoryException re) {
- RepositoryException me = (RepositoryException) re.getCause();
- Assert.assertEquals(me.getMessage(), "Invalid Id (unknown) : (type: Person, id: 3)");
-
- }
- }
-
- @Test
- public void testGetManager() throws MetadataException {
- TypeSystem ts = getTypeSystem();
- defineDeptEmployeeTypes(ts);
-
- Referenceable hrDept = createDeptEg1(ts);
- ITypedReferenceableInstance hrDept2 = getRepository().create(hrDept);
-
- Id m1Id = new Id(3, 0, "Manager");
- ITypedReferenceableInstance m1 = getRepository().get(m1Id);
- Assert.assertEquals(m1.toString(), "{\n" +
- "\tid : (type: Manager, id: 3)\n" +
- "\tsubordinates : \t[(type: Person, id: 2)]\n" +
- "\tname : \tJane\n" +
- "\tdepartment : (type: Department, id: 1)\n" +
- "\tmanager : <null>\n" +
- "\n" +
- "\tSecurityClearance : \t{\n" +
- "\t\tlevel : \t\t1\n" +
- "\t}}");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java
deleted file mode 100755
index 0e1247e..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/StructTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.BaseTest;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization$;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class StructTest extends BaseTest {
-
- StructType structType;
- StructType recursiveStructType;
-
- @Before
- public void setup() throws Exception {
- super.setup();
- structType = (StructType) getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
- recursiveStructType = (StructType) getTypeSystem()
- .getDataType(StructType.class, STRUCT_TYPE_2);
- }
-
- @Test
- public void test1() throws MetadataException {
- Struct s = createStruct();
- ITypedStruct ts = structType.convert(s, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t2\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.1, 1.1]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "}");
- }
-
- @Test
- public void testRecursive() throws MetadataException {
- Struct s1 = new Struct(recursiveStructType.getName());
- s1.set("a", 1);
- Struct s2 = new Struct(recursiveStructType.getName());
- s2.set("a", 1);
- s2.set("s", s1);
- ITypedStruct ts = recursiveStructType.convert(s2, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\ta : \t1\n" +
- "\ts : \t{\n" +
- "\t\ta : \t\t1\n" +
- "\t\ts : <null>\n" +
- "\n" +
- "\t}\n" +
- "}");
- }
-
- @Test
- public void testSerialization() throws MetadataException {
- Struct s = createStruct();
- String jsonStr = InstanceSerialization$.MODULE$.toJson(s, true);
- Struct s1 = InstanceSerialization$.MODULE$.fromJsonStruct(jsonStr, true);
- ITypedStruct ts = structType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\ta : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\td : \t2\n" +
- "\te : \t1\n" +
- "\tf : \t1\n" +
- "\tg : \t1\n" +
- "\th : \t1.0\n" +
- "\ti : \t1.0\n" +
- "\tj : \t1\n" +
- "\tk : \t1\n" +
- "\tl : \t" + TEST_DATE + "\n" +
- "\tm : \t[1, 1]\n" +
- "\tn : \t[1.100000000000000088817841970012523233890533447265625, 1" +
- ".100000000000000088817841970012523233890533447265625]\n" +
- "\to : \t{b=2.0, a=1.0}\n" +
- "}");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/TraitTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/TraitTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/TraitTest.java
deleted file mode 100755
index 02dde78..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/memory/TraitTest.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.memory;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.BaseTest;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-
-public class TraitTest extends BaseTest {
-
-
- @Before
- public void setup() throws Exception {
- super.setup();
- }
-
- /*
- * Type Hierarchy is:
- * A(a,b,c,d)
- * B(b) extends A
- * C(c) extends A
- * D(d) extends B,C
- *
- * - There are a total of 11 fields in an instance of D
- * - an attribute that is hidden by a SubType can referenced by prefixing it with the
- * complete Path.
- * For e.g. the 'b' attribute in A (that is a superType for B) is hidden the 'b' attribute
- * in B.
- * So it is availabel by the name 'A.B.D.b'
- *
- * - Another way to set attributes is to cast. Casting a 'D' instance of 'B' makes the 'A.B.D
- * .b' attribute
- * available as 'A.B.b'. Casting one more time to an 'A' makes the 'A.B.b' attribute
- * available as 'b'.
- */
- @Test
- public void test1() throws MetadataException {
- HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
- HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
- HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
- HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
-
- defineTraits(A, B, C, D);
-
- TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
-
- Struct s1 = new Struct("D");
- s1.set("d", 1);
- s1.set("c", 1);
- s1.set("b", true);
- s1.set("a", 1);
- s1.set("A.B.D.b", true);
- s1.set("A.B.D.c", 2);
- s1.set("A.B.D.d", 2);
-
- s1.set("A.C.D.a", 3);
- s1.set("A.C.D.b", false);
- s1.set("A.C.D.c", 3);
- s1.set("A.C.D.d", 3);
-
-
- ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- /*
- * cast to B and set the 'b' attribute on A.
- */
- TraitType BType = (TraitType) getTypeSystem().getDataType(TraitType.class, "B");
- IStruct s2 = DType.castAs(ts, "B");
- s2.set("A.B.b", false);
-
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \tfalse\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- /*
- * cast again to A and set the 'b' attribute on A.
- */
- TraitType AType = (TraitType) getTypeSystem().getDataType(TraitType.class, "A");
- IStruct s3 = BType.castAs(s2, "A");
- s3.set("b", true);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
- }
-
- @Test
- public void testRandomOrder() throws MetadataException {
- HierarchicalTypeDefinition A = createTraitTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
- HierarchicalTypeDefinition B = createTraitTypeDef("B", ImmutableList.<String>of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
- HierarchicalTypeDefinition C = createTraitTypeDef("C", ImmutableList.<String>of("A"),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
- HierarchicalTypeDefinition D = createTraitTypeDef("D", ImmutableList.<String>of("B", "C"),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
-
- defineTraits(B, D, A, C);
-
- TraitType DType = (TraitType) getTypeSystem().getDataType(TraitType.class, "D");
-
- Struct s1 = new Struct("D");
- s1.set("d", 1);
- s1.set("c", 1);
- s1.set("b", true);
- s1.set("a", 1);
- s1.set("A.B.D.b", true);
- s1.set("A.B.D.c", 2);
- s1.set("A.B.D.d", 2);
-
- s1.set("A.C.D.a", 3);
- s1.set("A.C.D.b", false);
- s1.set("A.C.D.c", 3);
- s1.set("A.C.D.d", 3);
-
-
- ITypedStruct ts = DType.convert(s1, Multiplicity.REQUIRED);
- Assert.assertEquals(ts.toString(), "{\n" +
- "\td : \t1\n" +
- "\tb : \ttrue\n" +
- "\tc : \t1\n" +
- "\ta : \t1\n" +
- "\tA.B.D.b : \ttrue\n" +
- "\tA.B.D.c : \t2\n" +
- "\tA.B.D.d : \t2\n" +
- "\tA.C.D.a : \t3\n" +
- "\tA.C.D.b : \tfalse\n" +
- "\tA.C.D.c : \t3\n" +
- "\tA.C.D.d : \t3\n" +
- "}");
-
- }
-
-}
-
[08/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
new file mode 100755
index 0000000..ba4782c
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json
+
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.typesystem.types._
+import org.junit.{Assert, Test}
+
+class TypesSerializationTest extends BaseTest with TypeHelpers {
+
+ @Test def test1: Unit = {
+
+ val ts = getTypeSystem
+
+ val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+ optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+ ts.defineTypes(ImmutableList.of[StructTypeDefinition](sDef),
+ ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
+ ImmutableList.of[HierarchicalTypeDefinition[ClassType]]
+ )
+
+ val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE))
+ val B: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
+ val C: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
+ val D: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
+
+ defineTraits(ts, A, B, C, D)
+
+ ts.defineEnumType("HiveObjectType",
+ new EnumValue("GLOBAL", 1),
+ new EnumValue("DATABASE", 2),
+ new EnumValue("TABLE", 3),
+ new EnumValue("PARTITION", 4),
+ new EnumValue("COLUMN", 5))
+
+ ts.defineEnumType("PrincipalType",
+ new EnumValue("USER", 1),
+ new EnumValue("ROLE", 2),
+ new EnumValue("GROUP", 3))
+
+ ts.defineEnumType("TxnState",
+ new EnumValue("COMMITTED", 1),
+ new EnumValue("ABORTED", 2),
+ new EnumValue("OPEN", 3))
+
+ ts.defineEnumType("LockLevel",
+ new EnumValue("DB", 1),
+ new EnumValue("TABLE", 2),
+ new EnumValue("PARTITION", 3))
+
+ defineClassType(ts, createClassTypeDef("t4", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("enum1", ts.getDataType(classOf[EnumType], "HiveObjectType")),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("enum2", ts.getDataType(classOf[EnumType], "PrincipalType")),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("enum3", ts.getDataType(classOf[EnumType], "TxnState")),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ optionalAttr("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ optionalAttr("enum4", ts.getDataType(classOf[EnumType], "LockLevel"))))
+
+ val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, true, "department"))
+ val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ )
+ val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
+ new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, false, "manager")
+ )
+ val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
+ requiredAttr("level", DataTypes.INT_TYPE)
+ )
+ ts.defineTypes(ImmutableList.of[StructTypeDefinition],
+ ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
+ ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
+
+ val ser = TypesSerialization.toJson(ts, _ => true)
+
+ val typesDef1 = TypesSerialization.fromJson(ser)
+
+ val ts1 = TypeSystem.getInstance()
+ ts1.reset()
+
+ typesDef1.enumTypes.foreach(ts1.defineEnumType(_))
+
+ ts1.defineTypes(ImmutableList.copyOf(typesDef1.structTypes.toArray),
+ ImmutableList.copyOf(typesDef1.traitTypes.toArray),
+ ImmutableList.copyOf(typesDef1.classTypes.toArray)
+ )
+ val ser2 = TypesSerialization.toJson(ts1, _ => true)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+
+ Assert.assertEquals(typesDef1, typesDef2)
+ }
+
+ @Test def test2: Unit = {
+
+ val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+ optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+
+ val ser2 = TypesSerialization.toJson(sDef)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+
+ Assert.assertEquals(sDef, typesDef2.structTypes(0))
+ }
+
+ @Test def test3: Unit = {
+
+ val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+ optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+
+ val ser2 = TypesSerialization.toJson(sDef)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+
+ Assert.assertEquals(sDef, typesDef2.structTypes(0))
+ }
+
+ @Test def test4 : Unit = {
+
+ val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE))
+ val B: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
+ val C: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
+ val D: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
+
+ val typDefs = Seq(A,B,C,D)
+ typDefs.foreach { tDef =>
+ val ser2 = TypesSerialization.toJson(tDef, true)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+ Assert.assertEquals(tDef, typesDef2.traitTypes(0))
+
+ }
+ }
+
+ @Test def test5 : Unit = {
+ val e1 = new EnumTypeDefinition("HiveObjectType",
+ new EnumValue("GLOBAL", 1),
+ new EnumValue("DATABASE", 2),
+ new EnumValue("TABLE", 3),
+ new EnumValue("PARTITION", 4),
+ new EnumValue("COLUMN", 5))
+
+ val e2 = new EnumTypeDefinition("PrincipalType",
+ new EnumValue("USER", 1),
+ new EnumValue("ROLE", 2),
+ new EnumValue("GROUP", 3))
+
+ val e3 = new EnumTypeDefinition("TxnState",
+ new EnumValue("COMMITTED", 1),
+ new EnumValue("ABORTED", 2),
+ new EnumValue("OPEN", 3))
+
+ val e4 = new EnumTypeDefinition("LockLevel",
+ new EnumValue("DB", 1),
+ new EnumValue("TABLE", 2),
+ new EnumValue("PARTITION", 3))
+
+ val typDefs = Seq(e1,e2,e3,e4)
+ typDefs.foreach { tDef =>
+ val ser2 = TypesSerialization.toJson(tDef)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+ Assert.assertEquals(tDef, typesDef2.enumTypes(0))
+
+ }
+ }
+
+ @Test def test6 : Unit = {
+ val typDef = createClassTypeDef("t4", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("enum1", "HiveObjectType"),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("enum2", "PrincipalType"),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("enum3", "TxnState"),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.INT_TYPE),
+ optionalAttr("n", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ optionalAttr("enum4", "LockLevel"))
+
+ val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, true, "department"))
+ val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ )
+ val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
+ new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, false, "manager")
+ )
+
+ val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef)
+ typDefs.foreach { tDef =>
+ val ser2 = TypesSerialization.toJson(tDef, false)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+ Assert.assertEquals(tDef, typesDef2.classTypes(0))
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
deleted file mode 100644
index 496ebf9..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.TypesDef
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import org.scalatest.{BeforeAndAfter, FunSuite}
-
-abstract class BuilderTest extends FunSuite with BeforeAndAfter {
-
- var tDef : TypesDef = null
-
- before {
- TypeSystem.getInstance().reset()
-
- val b = new TypesBuilder
- import b._
-
- tDef = types {
-
- _trait("Dimension") {}
- _trait("PII") {}
- _trait("Metric") {}
- _trait("ETL") {}
- _trait("JdbcAccess") {}
-
- _class("DB") {
- "name" ~ (string, required, indexed, unique)
- "owner" ~ (string)
- "createTime" ~ (int)
- }
-
- _class("StorageDesc") {
- "inputFormat" ~ (string, required)
- "outputFormat" ~ (string, required)
- }
-
- _class("Column") {
- "name" ~ (string, required)
- "dataType" ~ (string, required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("Table", List()) {
- "name" ~ (string, required, indexed)
- "db" ~ ("DB", required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("LoadProcess") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- "outputTable" ~ ("Table", required)
-
- }
-
- _class("View") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
deleted file mode 100644
index 227b524..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.types.{Multiplicity, ClassType, TypeSystem}
-
-
-class InstanceBuilderTest extends BuilderTest {
-
- test("test1") {
- TypeSystem.getInstance().defineTypes(tDef)
-
- val b = new InstanceBuilder
- import b._
-
- val instances = b create {
-
- val salesDB = instance("DB") { // use instance to create Referenceables. use closure to
- // set attributes of instance
- 'name ~ "Sales" // use '~' to set attributes. Use a Symbol (names starting with ') for
- // attribute names.
- 'owner ~ "John ETL"
- 'createTime ~ 1000
- }
-
- val salesFact = instance("Table") {
- 'name ~ "sales_fact"
- 'db ~ salesDB
- val sd = instance("StorageDesc") { // any valid scala allowed in closure.
- 'inputFormat ~ "TextIputFormat"
- 'outputFormat ~ "TextOutputFormat"
- }
- 'sd ~ sd // use ~ to set references, collections and maps.
- val columns = Seq(
- instance("Column") {
- 'name ~ "time_id"
- 'dataType ~ "int"
- 'sd ~ sd
- },
- instance("Column") {
- 'name ~ "product_id"
- 'dataType ~ "int"
- 'sd ~ sd
- },
- instance("Column") {
- 'name ~ "customer_id"
- 'dataType ~ "int"
- 'sd ~ sd
- },
- instance("Column", "Metric") {
- 'name ~ "sales"
- 'dataType ~ "int"
- 'sd ~ sd
- 'Metric("x") ~ 1 // use 'TraitName("attrName") to set values on traits.
- }
- )
-
- 'columns ~ columns
-
- }
-
- salesFact.sd.inputFormat ~ "TextInputFormat" // use dot navigation to alter attributes in the object graph.
- // here I am fixing the typo in "TextInputFormat"
- // dot navigation also works for arrays.
- // here I am fixing column(3). Metric trait has no attributes.
- val c = salesFact.columns
- c(3) = instance("Column", "Metric") {
- 'name ~ "sales"
- 'dataType ~ "int"
- 'sd ~ salesFact.sd
- }
-
- }
-
- val ts = TypeSystem.getInstance()
-
- import scala.collection.JavaConversions._
- val typedInstances = instances.map { i =>
- val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
- iTyp.convert(i, Multiplicity.REQUIRED)
- }
-
- typedInstances.foreach { i =>
- println(i)
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
deleted file mode 100644
index 73d912b..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.MetadataException
-import org.apache.hadoop.metadata.typesystem.types.{Multiplicity, ClassType, TypeSystem}
-import org.scalatest.{BeforeAndAfterAll, FunSuite}
-
-
-class MultiplicityTest extends FunSuite with BeforeAndAfterAll {
-
- override def beforeAll() = {
- TypeSystem.getInstance().reset()
-
- val b = new TypesBuilder
- import b._
-
- val tDef = types {
-
- _trait("Dimension") {}
- _trait("PII") {}
- _trait("Metric") {}
- _trait("ETL") {}
- _trait("JdbcAccess") {}
-
- _class("DB") {
- "name" ~ (string, required, indexed, unique)
- "owner" ~ (string)
- "createTime" ~ (int)
- }
-
- _class("StorageDesc") {
- "inputFormat" ~ (string, required)
- "outputFormat" ~ (string, required)
- }
-
- _class("Column") {
- "name" ~ (string, required)
- "dataType" ~ (string, required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("Table", List()) {
- "name" ~ (string, required, indexed)
- "db" ~ ("DB", required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("LoadProcess") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- "outputTable" ~ ("Table", required)
-
- }
-
- _class("View") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- }
-
- _class("AT") {
- "name" ~ (string, required)
- "stringSet" ~ (array("string"), multiplicty(0, Int.MaxValue, true))
- }
- }
-
- TypeSystem.getInstance().defineTypes(tDef)
- }
-
- test("test1") {
-
- val b = new InstanceBuilder
- import b._
-
- val instances = b create {
- val a = instance("AT") { // use instance to create Referenceables. use closure to
- // set attributes of instance
- 'name ~ "A1" // use '~' to set attributes. Use a Symbol (names starting with ') for
- 'stringSet ~ Seq("a", "a")
- }
- }
-
- val ts = TypeSystem.getInstance()
- import scala.collection.JavaConversions._
- val typedInstances = instances.map { i =>
- val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
- iTyp.convert(i, Multiplicity.REQUIRED)
- }
-
- typedInstances.foreach { i =>
- println(i)
- }
- }
-
- test("WrongMultiplicity") {
- val b = new TypesBuilder
- import b._
- val tDef = types {
- _class("Wrong") {
- "name" ~ (string, required)
- "stringSet" ~ (string, multiplicty(0, Int.MaxValue, true))
- }
- }
- val me = intercept[MetadataException] {
- TypeSystem.getInstance().defineTypes(tDef)
- }
- assert("A multiplicty of more than one requires a collection type for attribute 'stringSet'" == me.getMessage)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
deleted file mode 100644
index f1f9024..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization
-import org.apache.hadoop.metadata.typesystem.types.{TypeSystem, BaseTest}
-import org.junit.runner.RunWith
-import org.scalatest.{BeforeAndAfter, FunSuite}
-import org.scalatest.junit.JUnitRunner
-
-@RunWith(classOf[JUnitRunner])
-class TypesBuilderTest extends BuilderTest {
-
-
- test("test1") {
- TypeSystem.getInstance().defineTypes(tDef)
-
- println(TypesSerialization.toJson(TypeSystem.getInstance(), x => true))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
deleted file mode 100755
index 86cdb72..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.persistence.{ReferenceableInstance, StructInstance, Id}
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil
-import org.apache.hadoop.metadata.typesystem.{ITypedReferenceableInstance, ITypedStruct, Referenceable, Struct}
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.{write => swrite, _}
-import org.json4s.{NoTypeHints, _}
-import org.junit.{Assert, Before, Test}
-
-class SerializationTest extends BaseTest {
-
- private[metadata] var structType: StructType = null
- private[metadata] var recursiveStructType: StructType = null
-
- @Before
- override def setup {
- super.setup
- structType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_1).asInstanceOf[StructType]
- recursiveStructType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_2).asInstanceOf[StructType]
- }
-
- @Test def test1 {
- val s: Struct = BaseTest.createStruct()
- val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
-
- Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer
-
- //Json representation
- val ser = swrite(ts)
- Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"" + BaseTest.TEST_DATE + "\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
-
- // Typed Struct read back
- val ts1 = read[StructInstance](ser)
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
- }
-
- @Test def test2 {
- val s: Struct = BaseTest.createStruct()
- val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer
-
- val ts1 = read[StructInstance](
- """
- {"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
- "c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}""")
- // Typed Struct read from string
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
- }
-
- @Test def testTrait {
- val A: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef("A", null,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
- val B: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
- "B", ImmutableList.of[String]("A"),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE))
- val C: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
- "C", ImmutableList.of[String]("A"),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE))
- val D: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
- "D", ImmutableList.of[String]("B", "C"),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
-
- defineTraits(A, B, C, D)
-
- val DType: TraitType = getTypeSystem.getDataType(classOf[TraitType], "D").asInstanceOf[TraitType]
- val s1: Struct = new Struct("D")
- s1.set("d", 1)
- s1.set("c", 1)
- s1.set("b", true)
- s1.set("a", 1)
- s1.set("A.B.D.b", true)
- s1.set("A.B.D.c", 2)
- s1.set("A.B.D.d", 2)
- s1.set("A.C.D.a", 3)
- s1.set("A.C.D.b", false)
- s1.set("A.C.D.c", 3)
- s1.set("A.C.D.d", 3)
-
- val s: Struct = BaseTest.createStruct()
- val ts: ITypedStruct = DType.convert(s1, Multiplicity.REQUIRED)
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer
-
- // Typed Struct :
- Assert.assertEquals(ts.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
-
- // Json representation :
- val ser = swrite(ts)
- Assert.assertEquals(ser, "{\"$typeName$\":\"D\",\"A.C.D.d\":3,\"A.B.D.c\":2,\"b\":true,\"A.C.D.c\":3,\"d\":1,\"A.B.D.b\":true,\"a\":1,\"A.C.D.b\":false,\"A.B.D.d\":2,\"c\":1,\"A.C.D.a\":3}")
-
- val ts1 = read[StructInstance](
- """
- {"$typeName$":"D","A.C.D.d":3,"A.B.D.c":2,"b":true,"A.C.D.c":3,"d":1,
- "A.B.D.b":true,"a":1,"A.C.D.b":false,"A.B.D.d":2,"c":1,"A.C.D.a":3}""")
- // Typed Struct read from string:
- Assert.assertEquals(ts1.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
- }
-
- def defineHRTypes(ts: TypeSystem) : Unit = {
- val deptTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
- "Department",
- ImmutableList.of[String],
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, true, "department"))
- val personTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
- "Person", ImmutableList.of[String],
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"))
- val managerTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
- "Manager", ImmutableList.of[String]("Person"),
- new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager"))
- val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] =
- TypesUtil.createTraitTypeDef("SecurityClearance", ImmutableList.of[String],
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE))
-
- ts.defineTypes(ImmutableList.of[StructTypeDefinition],
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef)
- )
-
- }
-
- def defineHRDept() : Referenceable = {
- val hrDept: Referenceable = new Referenceable("Department")
- val john: Referenceable = new Referenceable("Person")
- val jane: Referenceable = new Referenceable("Manager", "SecurityClearance")
- hrDept.set("name", "hr")
- john.set("name", "John")
- john.set("department", hrDept.getId)
- jane.set("name", "Jane")
- jane.set("department", hrDept.getId)
- john.set("manager", jane.getId)
- hrDept.set("employees", ImmutableList.of[Referenceable](john, jane))
- jane.set("subordinates", ImmutableList.of[Id](john.getId))
- jane.getTrait("SecurityClearance").set("level", 1)
- hrDept
- }
-
- @Test def testClass {
-
- val ts: TypeSystem = getTypeSystem
- defineHRTypes(ts)
- val hrDept: Referenceable = defineHRDept()
-
- val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
- val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
-
- println(s"HR Dept Object Graph:\n${hrDept2}\n")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- val ser = swrite(hrDept2)
- println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
- println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
- }
-
- @Test def testReference {
-
- val ts: TypeSystem = getTypeSystem
- defineHRTypes(ts)
- val hrDept: Referenceable = defineHRDept()
-
-
- val jsonStr = InstanceSerialization.toJson(hrDept)
- val hrDept2 = InstanceSerialization.fromJsonReferenceable(jsonStr)
-
- val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
- val hrDept3: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
-
- println(s"HR Dept Object Graph:\n${hrDept3}\n")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- val ser = swrite(hrDept3)
- println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
- println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
- }
-
- @Test def testReference2 {
-
- val ts: TypeSystem = getTypeSystem
- defineHRTypes(ts)
- val hrDept: Referenceable = defineHRDept()
-
- val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
- val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
-
- val jsonStr = InstanceSerialization.toJson(hrDept2)
- val hrDept3 = InstanceSerialization.fromJsonReferenceable(jsonStr)
-
- val hrDept4: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
-
- println(s"HR Dept Object Graph:\n${hrDept4}\n")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- val ser = swrite(hrDept4)
- println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
- println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
deleted file mode 100755
index 7b4e44d..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
+++ /dev/null
@@ -1,300 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.types._
-import org.junit.{Assert, Test}
-
-class TypesSerializationTest extends BaseTest with TypeHelpers {
-
- @Test def test1: Unit = {
-
- val ts = getTypeSystem
-
- val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
- optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
- ts.defineTypes(ImmutableList.of[StructTypeDefinition](sDef),
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]]
- )
-
- val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE))
- val B: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
- val C: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
- val D: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
-
- defineTraits(ts, A, B, C, D)
-
- ts.defineEnumType("HiveObjectType",
- new EnumValue("GLOBAL", 1),
- new EnumValue("DATABASE", 2),
- new EnumValue("TABLE", 3),
- new EnumValue("PARTITION", 4),
- new EnumValue("COLUMN", 5))
-
- ts.defineEnumType("PrincipalType",
- new EnumValue("USER", 1),
- new EnumValue("ROLE", 2),
- new EnumValue("GROUP", 3))
-
- ts.defineEnumType("TxnState",
- new EnumValue("COMMITTED", 1),
- new EnumValue("ABORTED", 2),
- new EnumValue("OPEN", 3))
-
- ts.defineEnumType("LockLevel",
- new EnumValue("DB", 1),
- new EnumValue("TABLE", 2),
- new EnumValue("PARTITION", 3))
-
- defineClassType(ts, createClassTypeDef("t4", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("enum1", ts.getDataType(classOf[EnumType], "HiveObjectType")),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("enum2", ts.getDataType(classOf[EnumType], "PrincipalType")),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("enum3", ts.getDataType(classOf[EnumType], "TxnState")),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- optionalAttr("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- optionalAttr("enum4", ts.getDataType(classOf[EnumType], "LockLevel"))))
-
- val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, true, "department"))
- val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- )
- val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
- new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager")
- )
- val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
- requiredAttr("level", DataTypes.INT_TYPE)
- )
- ts.defineTypes(ImmutableList.of[StructTypeDefinition],
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
-
- val ser = TypesSerialization.toJson(ts, _ => true)
-
- val typesDef1 = TypesSerialization.fromJson(ser)
-
- val ts1 = TypeSystem.getInstance()
- ts1.reset()
-
- typesDef1.enumTypes.foreach(ts1.defineEnumType(_))
-
- ts1.defineTypes(ImmutableList.copyOf(typesDef1.structTypes.toArray),
- ImmutableList.copyOf(typesDef1.traitTypes.toArray),
- ImmutableList.copyOf(typesDef1.classTypes.toArray)
- )
- val ser2 = TypesSerialization.toJson(ts1, _ => true)
- val typesDef2 = TypesSerialization.fromJson(ser2)
-
- Assert.assertEquals(typesDef1, typesDef2)
- }
-
- @Test def test2: Unit = {
-
- val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
- optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
-
- val ser2 = TypesSerialization.toJson(sDef)
- val typesDef2 = TypesSerialization.fromJson(ser2)
-
- Assert.assertEquals(sDef, typesDef2.structTypes(0))
- }
-
- @Test def test3: Unit = {
-
- val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
- optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
-
- val ser2 = TypesSerialization.toJson(sDef)
- val typesDef2 = TypesSerialization.fromJson(ser2)
-
- Assert.assertEquals(sDef, typesDef2.structTypes(0))
- }
-
- @Test def test4 : Unit = {
-
- val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE))
- val B: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
- val C: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
- val D: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
-
- val typDefs = Seq(A,B,C,D)
- typDefs.foreach { tDef =>
- val ser2 = TypesSerialization.toJson(tDef, true)
- val typesDef2 = TypesSerialization.fromJson(ser2)
- Assert.assertEquals(tDef, typesDef2.traitTypes(0))
-
- }
- }
-
- @Test def test5 : Unit = {
- val e1 = new EnumTypeDefinition("HiveObjectType",
- new EnumValue("GLOBAL", 1),
- new EnumValue("DATABASE", 2),
- new EnumValue("TABLE", 3),
- new EnumValue("PARTITION", 4),
- new EnumValue("COLUMN", 5))
-
- val e2 = new EnumTypeDefinition("PrincipalType",
- new EnumValue("USER", 1),
- new EnumValue("ROLE", 2),
- new EnumValue("GROUP", 3))
-
- val e3 = new EnumTypeDefinition("TxnState",
- new EnumValue("COMMITTED", 1),
- new EnumValue("ABORTED", 2),
- new EnumValue("OPEN", 3))
-
- val e4 = new EnumTypeDefinition("LockLevel",
- new EnumValue("DB", 1),
- new EnumValue("TABLE", 2),
- new EnumValue("PARTITION", 3))
-
- val typDefs = Seq(e1,e2,e3,e4)
- typDefs.foreach { tDef =>
- val ser2 = TypesSerialization.toJson(tDef)
- val typesDef2 = TypesSerialization.fromJson(ser2)
- Assert.assertEquals(tDef, typesDef2.enumTypes(0))
-
- }
- }
-
- @Test def test6 : Unit = {
- val typDef = createClassTypeDef("t4", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("enum1", "HiveObjectType"),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("enum2", "PrincipalType"),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("enum3", "TxnState"),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.INT_TYPE),
- optionalAttr("n", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- optionalAttr("enum4", "LockLevel"))
-
- val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, true, "department"))
- val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- )
- val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
- new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager")
- )
-
- val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef)
- typDefs.foreach { tDef =>
- val ser2 = TypesSerialization.toJson(tDef, false)
- val typesDef2 = TypesSerialization.fromJson(ser2)
- Assert.assertEquals(tDef, typesDef2.classTypes(0))
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/Main.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/Main.java b/webapp/src/main/java/org/apache/atlas/Main.java
new file mode 100755
index 0000000..76ca965
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/Main.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import org.apache.atlas.web.service.EmbeddedServer;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+
+/**
+ * Driver for running Metadata as a standalone server with embedded jetty server.
+ */
+public final class Main {
+ private static final Logger LOG = LoggerFactory.getLogger(Main.class);
+ private static final String APP_PATH = "app";
+ private static final String APP_PORT = "port";
+ private static final String METADATA_HOME = "atlas.home";
+ private static final String METADATA_LOG_DIR = "atlas.log.dir";
+
+ /**
+ * Prevent users from constructing this.
+ */
+ private Main() {
+ }
+
+ private static CommandLine parseArgs(String[] args) throws ParseException {
+ Options options = new Options();
+ Option opt;
+
+ opt = new Option(APP_PATH, true, "Application Path");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option(APP_PORT, true, "Application Port");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ return new GnuParser().parse(options, args);
+ }
+
+ public static void main(String[] args) throws Exception {
+ CommandLine cmd = parseArgs(args);
+ PropertiesConfiguration buildConfiguration =
+ new PropertiesConfiguration("atlas-buildinfo.properties");
+ String appPath = "webapp/target/atlas-webapp-" + getProjectVersion(buildConfiguration);
+
+ if (cmd.hasOption(APP_PATH)) {
+ appPath = cmd.getOptionValue(APP_PATH);
+ }
+
+ setApplicationHome();
+ PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
+ final String enableTLSFlag = configuration.getString("atlas.enableTLS");
+ final int appPort = getApplicationPort(cmd, enableTLSFlag);
+ final boolean enableTLS = isTLSEnabled(enableTLSFlag, appPort);
+ configuration.setProperty("atlas.enableTLS", String.valueOf(enableTLS));
+
+ showStartupInfo(buildConfiguration, enableTLS, appPort);
+ EmbeddedServer server = EmbeddedServer.newServer(appPort, appPath, enableTLS);
+ server.start();
+ }
+
+ private static void setApplicationHome() {
+ if (System.getProperty(METADATA_HOME) == null) {
+ System.setProperty(METADATA_HOME, "target");
+ }
+ if (System.getProperty(METADATA_LOG_DIR) == null) {
+ System.setProperty(METADATA_LOG_DIR, "target/logs");
+ }
+ }
+
+ public static String getProjectVersion(PropertiesConfiguration buildConfiguration) {
+ return buildConfiguration.getString("project.version");
+ }
+
+ private static int getApplicationPort(CommandLine cmd, String enableTLSFlag) {
+ final int appPort;
+ if (cmd.hasOption(APP_PORT)) {
+ appPort = Integer.valueOf(cmd.getOptionValue(APP_PORT));
+ } else {
+ // default : metadata.enableTLS is true
+ appPort = StringUtils.isEmpty(enableTLSFlag)
+ || enableTLSFlag.equals("true") ? 21443 : 21000;
+ }
+
+ return appPort;
+ }
+
+ private static boolean isTLSEnabled(String enableTLSFlag, int appPort) {
+ return Boolean.valueOf(StringUtils.isEmpty(enableTLSFlag)
+ ? System
+ .getProperty("atlas.enableTLS", (appPort % 1000) == 443 ? "true" : "false")
+ : enableTLSFlag);
+ }
+
+ private static void showStartupInfo(PropertiesConfiguration buildConfiguration,
+ boolean enableTLS, int appPort) {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("\n############################################");
+ buffer.append("############################################");
+ buffer.append("\n DGI Server (STARTUP)");
+ buffer.append("\n");
+ try {
+ final Iterator<String> keys = buildConfiguration.getKeys();
+ while (keys.hasNext()) {
+ String key = keys.next();
+ buffer.append('\n').append('\t').append(key).
+ append(":\t").append(buildConfiguration.getProperty(key));
+ }
+ } catch (Throwable e) {
+ buffer.append("*** Unable to get build info ***");
+ }
+ buffer.append("\n############################################");
+ buffer.append("############################################");
+ LOG.info(buffer.toString());
+ LOG.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ LOG.info("Server starting with TLS ? {} on port {}", enableTLS, appPort);
+ LOG.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
new file mode 100755
index 0000000..4c0ffe8
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
@@ -0,0 +1,478 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.examples;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+
+/**
+ * A driver that sets up sample types and data for testing purposes.
+ * Please take a look at QueryDSL in docs for the Meta Model.
+ * todo - move this to examples module.
+ */
+public class QuickStart {
+
+ public static void main(String[] args) throws Exception {
+ String baseUrl = getServerUrl(args);
+ QuickStart quickStart = new QuickStart(baseUrl);
+
+ // Shows how to create types in DGI for your meta model
+ quickStart.createTypes();
+
+ // Shows how to create entities (instances) for the added types in DGI
+ quickStart.createEntities();
+
+ // Shows some search queries using DSL based on types
+ quickStart.search();
+ }
+
+ static String getServerUrl(String[] args) {
+ String baseUrl = "http://localhost:21000";
+ if (args.length > 0) {
+ baseUrl = args[0];
+ }
+
+ return baseUrl;
+ }
+
+ private static final String DATABASE_TYPE = "DB";
+ private static final String COLUMN_TYPE = "Column";
+ private static final String TABLE_TYPE = "Table";
+ private static final String VIEW_TYPE = "View";
+ private static final String LOAD_PROCESS_TYPE = "LoadProcess";
+ private static final String STORAGE_DESC_TYPE = "StorageDesc";
+
+ private static final String[] TYPES = {
+ DATABASE_TYPE, TABLE_TYPE, STORAGE_DESC_TYPE, COLUMN_TYPE, LOAD_PROCESS_TYPE, VIEW_TYPE,
+ "JdbcAccess", "ETL", "Metric", "PII", "Fact", "Dimension"
+ };
+
+ private final MetadataServiceClient metadataServiceClient;
+
+ QuickStart(String baseUrl) {
+ metadataServiceClient = new MetadataServiceClient(baseUrl);
+ }
+
+ void createTypes() throws Exception {
+ TypesDef typesDef = createTypeDefinitions();
+
+ String typesAsJSON = TypesSerialization.toJson(typesDef);
+ System.out.println("typesAsJSON = " + typesAsJSON);
+ metadataServiceClient.createType(typesAsJSON);
+
+ // verify types created
+ verifyTypesCreated();
+ }
+
+ TypesDef createTypeDefinitions() throws Exception {
+ HierarchicalTypeDefinition<ClassType> dbClsDef
+ = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("description", DataTypes.STRING_TYPE),
+ attrDef("locationUri", DataTypes.STRING_TYPE),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.INT_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> storageDescClsDef =
+ TypesUtil.createClassTypeDef(STORAGE_DESC_TYPE, null,
+ attrDef("location", DataTypes.STRING_TYPE),
+ attrDef("inputFormat", DataTypes.STRING_TYPE),
+ attrDef("outputFormat", DataTypes.STRING_TYPE),
+ attrDef("compressed", DataTypes.STRING_TYPE,
+ Multiplicity.REQUIRED, false, null)
+ );
+
+ HierarchicalTypeDefinition<ClassType> columnClsDef =
+ TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("dataType", DataTypes.STRING_TYPE),
+ attrDef("comment", DataTypes.STRING_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> tblClsDef =
+ TypesUtil.createClassTypeDef(TABLE_TYPE, ImmutableList.of("DataSet"),
+ new AttributeDefinition("db", DATABASE_TYPE,
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("sd", STORAGE_DESC_TYPE,
+ Multiplicity.REQUIRED, true, null),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.INT_TYPE),
+ attrDef("lastAccessTime", DataTypes.INT_TYPE),
+ attrDef("retention", DataTypes.INT_TYPE),
+ attrDef("viewOriginalText", DataTypes.STRING_TYPE),
+ attrDef("viewExpandedText", DataTypes.STRING_TYPE),
+ attrDef("tableType", DataTypes.STRING_TYPE),
+ attrDef("temporary", DataTypes.BOOLEAN_TYPE),
+ new AttributeDefinition("columns",
+ DataTypes.arrayTypeName(COLUMN_TYPE),
+ Multiplicity.COLLECTION, true, null)
+ );
+
+ HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
+ TypesUtil.createClassTypeDef(LOAD_PROCESS_TYPE, ImmutableList.of("Process"),
+ attrDef("userName", DataTypes.STRING_TYPE),
+ attrDef("startTime", DataTypes.INT_TYPE),
+ attrDef("endTime", DataTypes.INT_TYPE),
+ attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
+ );
+
+ HierarchicalTypeDefinition<ClassType> viewClsDef =
+ TypesUtil.createClassTypeDef(VIEW_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("db", DATABASE_TYPE,
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("inputTables",
+ DataTypes.arrayTypeName(TABLE_TYPE),
+ Multiplicity.COLLECTION, false, null)
+ );
+
+ HierarchicalTypeDefinition<TraitType> dimTraitDef =
+ TypesUtil.createTraitTypeDef("Dimension", null);
+
+ HierarchicalTypeDefinition<TraitType> factTraitDef =
+ TypesUtil.createTraitTypeDef("Fact", null);
+
+ HierarchicalTypeDefinition<TraitType> piiTraitDef =
+ TypesUtil.createTraitTypeDef("PII", null);
+
+ HierarchicalTypeDefinition<TraitType> metricTraitDef =
+ TypesUtil.createTraitTypeDef("Metric", null);
+
+ HierarchicalTypeDefinition<TraitType> etlTraitDef =
+ TypesUtil.createTraitTypeDef("ETL", null);
+
+ HierarchicalTypeDefinition<TraitType> jdbcTraitDef =
+ TypesUtil.createTraitTypeDef("JdbcAccess", null);
+
+ return TypeUtils.getTypesDef(
+ ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(dimTraitDef, factTraitDef,
+ piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
+ ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef,
+ tblClsDef, loadProcessClsDef, viewClsDef)
+ );
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT) {
+ return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
+ return attrDef(name, dT, m, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT,
+ Multiplicity m, boolean isComposite, String reverseAttributeName) {
+ Preconditions.checkNotNull(name);
+ Preconditions.checkNotNull(dT);
+ return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
+ }
+
+ void createEntities() throws Exception {
+ Id salesDB = database(
+ "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
+
+
+ Referenceable sd = rawStorageDescriptor("hdfs://host:8000/apps/warehouse/sales",
+ "TextInputFormat", "TextOutputFormat", true);
+
+ List<Referenceable> salesFactColumns = ImmutableList.of(
+ rawColumn("time_id", "int", "time id"),
+ rawColumn("product_id", "int", "product id"),
+ rawColumn("customer_id", "int", "customer id", "PII"),
+ rawColumn("sales", "double", "product id", "Metric")
+ );
+
+ Id salesFact = table("sales_fact", "sales fact table",
+ salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
+
+ List<Referenceable> productDimColumns = ImmutableList.of(
+ rawColumn("product_id", "int", "product id"),
+ rawColumn("product_name", "string", "product name"),
+ rawColumn("brand_name", "int", "brand name")
+ );
+
+ Id productDim = table("product_dim", "product dimension table",
+ salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
+
+ List<Referenceable> timeDimColumns = ImmutableList.of(
+ rawColumn("time_id", "int", "time id"),
+ rawColumn("dayOfYear", "int", "day Of Year"),
+ rawColumn("weekDay", "int", "week Day")
+ );
+
+ Id timeDim = table("time_dim", "time dimension table",
+ salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
+
+
+ List<Referenceable> customerDimColumns = ImmutableList.of(
+ rawColumn("customer_id", "int", "customer id", "PII"),
+ rawColumn("name", "string", "customer name", "PII"),
+ rawColumn("address", "string", "customer address", "PII")
+ );
+
+ Id customerDim = table("customer_dim", "customer dimension table",
+ salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
+
+
+ Id reportingDB = database("Reporting", "reporting database", "Jane BI",
+ "hdfs://host:8000/apps/warehouse/reporting");
+
+ Id salesFactDaily = table("sales_fact_daily_mv",
+ "sales fact daily materialized view", reportingDB, sd,
+ "Joe BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL",
+ ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+
+ view("product_dim_view", reportingDB,
+ ImmutableList.of(productDim), "Dimension", "JdbcAccess");
+
+ view("customer_dim_view", reportingDB,
+ ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
+
+ Id salesFactMonthly = table("sales_fact_monthly_mv",
+ "sales fact monthly materialized view",
+ reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL",
+ ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+ }
+
+ private Id createInstance(Referenceable referenceable) throws Exception {
+ String typeName = referenceable.getTypeName();
+
+ String entityJSON = InstanceSerialization.toJson(referenceable, true);
+ System.out.println("Submitting new entity= " + entityJSON);
+ JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
+ String guid = jsonObject.getString(MetadataServiceClient.GUID);
+ System.out.println("created instance for type " + typeName + ", guid: " + guid);
+
+ // return the Id for created instance with guid
+ return new Id(guid, referenceable.getId().getVersion(), referenceable.getTypeName());
+ }
+
+ Id database(String name, String description,
+ String owner, String locationUri,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("locationUri", locationUri);
+ referenceable.set("createTime", System.currentTimeMillis());
+
+ return createInstance(referenceable);
+ }
+
+ Referenceable rawStorageDescriptor(String location, String inputFormat,
+ String outputFormat,
+ boolean compressed) throws Exception {
+ Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
+ referenceable.set("location", location);
+ referenceable.set("inputFormat", inputFormat);
+ referenceable.set("outputFormat", outputFormat);
+ referenceable.set("compressed", compressed);
+
+ return referenceable;
+ }
+
+ Referenceable rawColumn(String name, String dataType, String comment,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("dataType", dataType);
+ referenceable.set("comment", comment);
+
+ return referenceable;
+ }
+
+ Id table(String name, String description,
+ Id dbId, Referenceable sd,
+ String owner, String tableType,
+ List<Referenceable> columns,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(TABLE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("tableType", tableType);
+ referenceable.set("createTime", System.currentTimeMillis());
+ referenceable.set("lastAccessTime", System.currentTimeMillis());
+ referenceable.set("retention", System.currentTimeMillis());
+ referenceable.set("db", dbId);
+ referenceable.set("sd", sd);
+ referenceable.set("columns", columns);
+
+ return createInstance(referenceable);
+ }
+
+ Id loadProcess(String name, String description, String user,
+ List<Id> inputTables,
+ List<Id> outputTables,
+ String queryText, String queryPlan,
+ String queryId, String queryGraph,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(LOAD_PROCESS_TYPE, traitNames);
+ // super type attributes
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("inputs", inputTables);
+ referenceable.set("outputs", outputTables);
+
+ referenceable.set("user", user);
+ referenceable.set("startTime", System.currentTimeMillis());
+ referenceable.set("endTime", System.currentTimeMillis() + 10000);
+
+ referenceable.set("queryText", queryText);
+ referenceable.set("queryPlan", queryPlan);
+ referenceable.set("queryId", queryId);
+ referenceable.set("queryGraph", queryGraph);
+
+ return createInstance(referenceable);
+ }
+
+ Id view(String name, Id dbId,
+ List<Id> inputTables,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("db", dbId);
+
+ referenceable.set("inputTables", inputTables);
+
+ return createInstance(referenceable);
+ }
+
+ private void verifyTypesCreated() throws Exception {
+ List<String> types = metadataServiceClient.listTypes();
+ for (String type : TYPES) {
+ assert types.contains(type);
+ }
+ }
+
+ private String[] getDSLQueries() {
+ return new String[]{
+ "from DB",
+ "DB",
+ "DB where name=\"Reporting\"",
+ "DB where DB.name=\"Reporting\"",
+ "DB name = \"Reporting\"",
+ "DB DB.name = \"Reporting\"",
+ "DB where name=\"Reporting\" select name, owner",
+ "DB where DB.name=\"Reporting\" select name, owner",
+ "DB has name",
+ "DB where DB has name",
+ "DB, Table",
+ "DB is JdbcAccess",
+ /*
+ "DB, hive_process has name",
+ "DB as db1, Table where db1.name = \"Reporting\"",
+ "DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
+ */
+ "from Table",
+ "Table",
+ "Table is Dimension",
+ "Column where Column isa PII",
+ "View is Dimension",
+ /*"Column where Column isa PII select Column.name",*/
+ "Column select Column.name",
+ "Column select name",
+ "Column where Column.name=\"customer_id\"",
+ "from Table select Table.name",
+ "DB where (name = \"Reporting\")",
+ "DB where (name = \"Reporting\") select name as _col_0, owner as _col_1",
+ "DB where DB is JdbcAccess",
+ "DB where DB has name",
+ "DB Table",
+ "DB where DB has name",
+ "DB as db1 Table where (db1.name = \"Reporting\")",
+ "DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ",
+ /*
+ todo: does not work
+ "DB where (name = \"Reporting\") and ((createTime + 1) > 0)",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
+ */
+ // trait searches
+ "Dimension",
+ /*"Fact", - todo: does not work*/
+ "JdbcAccess",
+ "ETL",
+ "Metric",
+ "PII",
+ /*
+ // Lineage - todo - fix this, its not working
+ "Table hive_process outputTables",
+ "Table loop (hive_process outputTables)",
+ "Table as _loop0 loop (hive_process outputTables) withPath",
+ "Table as src loop (hive_process outputTables) as dest select src.name as srcTable, dest.name as destTable withPath",
+ */
+ "Table where name=\"sales_fact\", columns",
+ "Table where name=\"sales_fact\", columns as column select column.name, column.dataType, column.comment",
+ "from DataSet",
+ "from Process",
+ };
+ }
+
+ private void search() throws Exception {
+ for (String dslQuery : getDSLQueries()) {
+ JSONObject response = metadataServiceClient.searchEntity(dslQuery);
+ JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
+ if (!results.isNull("rows")) {
+ JSONArray rows = results.getJSONArray("rows");
+ System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
+ } else {
+ System.out.println("query [" + dslQuery + "] failed, results:" + results.toString());
+ }
+ }
+ }
+}
[56/58] [abbrv] incubator-atlas git commit: Merge pull request #11
from MPR-Global/HDPDGI-79
Posted by ve...@apache.org.
Merge pull request #11 from MPR-Global/HDPDGI-79
show arrow pointing inwards for input tab
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/da9f12a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/da9f12a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/da9f12a3
Branch: refs/heads/master
Commit: da9f12a334b2a5a00fee6675ecabc6763b309e52
Parents: eb9eb4c 2fa32ec
Author: mouly <mo...@users.noreply.github.com>
Authored: Mon Jun 15 21:56:12 2015 -0700
Committer: mouly <mo...@users.noreply.github.com>
Committed: Mon Jun 15 21:56:12 2015 -0700
----------------------------------------------------------------------
.../public/modules/lineage/lineageController.js | 21 +++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/da9f12a3/dashboard/v2/public/modules/lineage/lineageController.js
----------------------------------------------------------------------
diff --cc dashboard/v2/public/modules/lineage/lineageController.js
index 6539375,368e9a5..6e4e1e3
--- a/dashboard/v2/public/modules/lineage/lineageController.js
+++ b/dashboard/v2/public/modules/lineage/lineageController.js
@@@ -189,13 -130,26 +189,27 @@@ angular.module('dgc.lineage').controlle
var svg = element.select('svg')
.attr('width', width + margin.right + margin.left)
.attr('height', height + margin.top + margin.bottom)
+ /* Invoke the tip in the context of your visualization */
+ .call(tooltip)
.select('g')
-
- .attr('transform',
- 'translate(' + margin.left + ',' + margin.right + ')');
+ .attr('transform',
+ 'translate(' + margin.left + ',' + margin.right + ')');
//arrow
svg.append("svg:defs").append("svg:marker").attr("id", "arrow").attr("viewBox", "0 0 10 10").attr("refX", 26).attr("refY", 5).attr("markerUnits", "strokeWidth").attr("markerWidth", 6).attr("markerHeight", 9).attr("orient", "auto").append("svg:path").attr("d", "M 0 0 L 10 5 L 0 10 z");
+
+ //marker for input type graph
+ svg.append("svg:defs")
+ .append("svg:marker")
+ .attr("id", "input-arrow")
+ .attr("viewBox", "0 0 10 10")
+ .attr("refX", -15)
+ .attr("refY", 5)
+ .attr("markerUnits", "strokeWidth")
+ .attr("markerWidth", 6)
+ .attr("markerHeight", 9)
+ .attr("orient", "auto")
+ .append("svg:path")
+ .attr("d", "M -2 5 L 8 0 L 8 10 z");
var root = data;
[17/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
new file mode 100755
index 0000000..6f04fb8
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.persistence;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.types.FieldMapping;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.Map;
+import java.util.UUID;
+
+public class Id implements ITypedReferenceableInstance {
+
+ public final String id;
+ public final String className;
+ public final int version;
+
+ public Id(String id, int version, String className) {
+ this.id = id;
+ this.className = className;
+ this.version = version;
+ }
+
+ public Id(long id, int version, String className) {
+ this("" + id, version, className);
+ }
+
+ public Id(String className) {
+ this("" + (-System.nanoTime()), 0, className);
+ }
+
+ public boolean isUnassigned() {
+ try {
+ long l = Long.parseLong(id);
+ return l < 0;
+ } catch (NumberFormatException ne) {
+ return false;
+ }
+ }
+
+ public boolean isAssigned() {
+ try {
+ UUID.fromString(id);
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public String toString() {
+ return String
+ .format("(type: %s, id: %s)", className, isUnassigned() ? "<unassigned>" : "" + id);
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ public String _getId() {
+ return id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Id id1 = (Id) o;
+
+ if (version != id1.version) return false;
+ if (!className.equals(id1.className)) return false;
+ if (!id.equals(id1.id)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id.hashCode();
+ result = 31 * result + className.hashCode();
+ result = 31 * result + version;
+ return result;
+ }
+
+ @Override
+ public ImmutableList<String> getTraits() {
+ return null;
+ }
+
+ @Override
+ public Id getId() {
+ return this;
+ }
+
+ @Override
+ public IStruct getTrait(String typeName) {
+ return null;
+ }
+
+ @Override
+ public String getTypeName() {
+ return className;
+ }
+
+ @Override
+ public Object get(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ @Override
+ public void set(String attrName, Object val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ @Override
+ public FieldMapping fieldMapping() {
+ return null;
+ }
+
+ @Override
+ public Map<String, Object> getValuesMap() throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setNull(String attrName) throws MetadataException {
+ set(attrName, null);
+ }
+
+ public boolean getBoolean(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public byte getByte(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public short getShort(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public int getInt(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public long getLong(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public float getFloat(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public double getDouble(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public BigInteger getBigInt(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public BigDecimal getBigDecimal(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public Date getDate(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public String getString(String attrName) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setBoolean(String attrName, boolean val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setByte(String attrName, byte val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setShort(String attrName, short val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setInt(String attrName, int val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setLong(String attrName, long val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setFloat(String attrName, float val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setDouble(String attrName, double val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setBigInt(String attrName, BigInteger val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setBigDecimal(String attrName, BigDecimal val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setDate(String attrName, Date val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+
+ public void setString(String attrName, String val) throws MetadataException {
+ throw new MetadataException("Get/Set not supported on an Id object");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/MapIds.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/MapIds.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/MapIds.java
new file mode 100755
index 0000000..8dae5a3
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/MapIds.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.persistence;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+
+import java.util.Map;
+
+public class MapIds implements ObjectGraphWalker.NodeProcessor {
+
+ final Map<Id, Id> idToNewIdMap;
+
+ public MapIds(Map<Id, Id> idToNewIdMap) {
+ this.idToNewIdMap = idToNewIdMap;
+ }
+
+ @Override
+ public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
+
+ IReferenceableInstance ref = null;
+ Id id = null;
+
+ if (nd.attributeName == null) {
+ ref = (IReferenceableInstance) nd.instance;
+ Id newId = idToNewIdMap.get(ref.getId());
+ if (newId != null) {
+ ((ReferenceableInstance) ref).replaceWithNewId(newId);
+ }
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ if (nd.value != null && nd.value instanceof IReferenceableInstance) {
+ Id oldId = ((IReferenceableInstance) nd.value).getId();
+ Id newId = idToNewIdMap.get(oldId);
+ /*
+ * Replace Instances with Ids, irrespective of whether they map to newIds or not.
+ */
+ newId = newId == null ? oldId : newId;
+ nd.instance.set(nd.attributeName, newId);
+ }
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ DataTypes.ArrayType aT = (DataTypes.ArrayType) nd.aInfo.dataType();
+ Object v = aT
+ .mapIds((ImmutableCollection) nd.value, nd.aInfo.multiplicity, idToNewIdMap);
+ nd.instance.set(nd.attributeName, v);
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ DataTypes.MapType mT = (DataTypes.MapType) nd.aInfo.dataType();
+ Object v = mT.mapIds((ImmutableMap) nd.value, nd.aInfo.multiplicity, idToNewIdMap);
+ nd.instance.set(nd.attributeName, v);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
new file mode 100755
index 0000000..1ec8e07
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.persistence;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.types.FieldMapping;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+
+/*
+ * @todo handle names prefixed by traitName.
+ */
+public class ReferenceableInstance extends StructInstance implements ITypedReferenceableInstance {
+
+ private final ImmutableMap<String, ITypedStruct> traits;
+ private final ImmutableList<String> traitNames;
+ private Id id;
+
+
+ public ReferenceableInstance(Id id, String dataTypeName, FieldMapping fieldMapping,
+ boolean[] nullFlags,
+ boolean[] bools, byte[] bytes, short[] shorts, int[] ints,
+ long[] longs,
+ float[] floats, double[] doubles, BigDecimal[] bigDecimals,
+ BigInteger[] bigIntegers, Date[] dates, String[] strings,
+ ImmutableList<Object>[] arrays,
+ ImmutableMap<Object, Object>[] maps,
+ StructInstance[] structs,
+ ReferenceableInstance[] referenceableInstances,
+ Id[] ids,
+ ImmutableMap<String, ITypedStruct> traits) {
+ super(dataTypeName, fieldMapping, nullFlags, bools, bytes, shorts, ints, longs, floats,
+ doubles, bigDecimals,
+ bigIntegers, dates, strings, arrays, maps, structs, referenceableInstances, ids);
+ this.id = id;
+ this.traits = traits;
+ ImmutableList.Builder<String> b = new ImmutableList.Builder<String>();
+ for (String t : traits.keySet()) {
+ b.add(t);
+ }
+ this.traitNames = b.build();
+ }
+
+ @Override
+ public ImmutableList<String> getTraits() {
+ return traitNames;
+ }
+
+ @Override
+ public Id getId() {
+ return id;
+ }
+
+ @Override
+ public IStruct getTrait(String typeName) {
+ return traits.get(typeName);
+ }
+
+ /**
+ * @nopub
+ * @param id
+ */
+ void replaceWithNewId(Id id) {
+ this.id = id;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ StringBuilder buf = new StringBuilder();
+ String prefix = "";
+
+ fieldMapping.output(this, buf, prefix);
+ return buf.toString();
+
+ } catch (MetadataException me) {
+ throw new RuntimeException(me);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
new file mode 100755
index 0000000..cf89bb0
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
@@ -0,0 +1,757 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.persistence;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.FieldMapping;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.ValueConversionException;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+public class StructInstance implements ITypedStruct {
+ public final String dataTypeName;
+ public final FieldMapping fieldMapping;
+ public final boolean nullFlags[];
+ public final boolean[] bools;
+ public final byte[] bytes;
+ public final short[] shorts;
+ public final int[] ints;
+ public final long[] longs;
+ public final float[] floats;
+ public final double[] doubles;
+ public final BigDecimal[] bigDecimals;
+ public final BigInteger[] bigIntegers;
+ public final Date[] dates;
+ public final String[] strings;
+ public final ImmutableList<Object>[] arrays;
+ public final ImmutableMap<Object, Object>[] maps;
+ public final StructInstance[] structs;
+ public final ReferenceableInstance[] referenceables;
+ public final Id[] ids;
+
+ public StructInstance(String dataTypeName, FieldMapping fieldMapping,
+ boolean[] nullFlags, boolean[] bools, byte[] bytes, short[] shorts,
+ int[] ints,
+ long[] longs, float[] floats, double[] doubles,
+ BigDecimal[] bigDecimals, BigInteger[] bigIntegers, Date[] dates,
+ String[] strings,
+ ImmutableList<Object>[] arrays, ImmutableMap<Object, Object>[] maps,
+ StructInstance[] structs, ReferenceableInstance[] referenceables,
+ Id[] ids) {
+ assert dataTypeName != null;
+ this.dataTypeName = dataTypeName;
+ this.fieldMapping = fieldMapping;
+ this.nullFlags = nullFlags;
+ this.bools = bools;
+ this.bytes = bytes;
+ this.shorts = shorts;
+ this.ints = ints;
+ this.longs = longs;
+ this.floats = floats;
+ this.doubles = doubles;
+ this.bigDecimals = bigDecimals;
+ this.bigIntegers = bigIntegers;
+ this.dates = dates;
+ this.strings = strings;
+ this.arrays = arrays;
+ this.maps = maps;
+ this.structs = structs;
+ this.referenceables = referenceables;
+ this.ids = ids;
+
+ for (int i = 0; i < nullFlags.length; i++) {
+ nullFlags[i] = true;
+ }
+ }
+
+ @Override
+ public String getTypeName() {
+ return dataTypeName;
+ }
+
+ @Override
+ public FieldMapping fieldMapping() {
+ return fieldMapping;
+ }
+
+ public void set(String attrName, Object val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new ValueConversionException(getTypeName(), val, "Unknown field " + attrName);
+ }
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+ Object cVal = null;
+
+ if (val != null && val instanceof Id) {
+ ClassType clsType =
+ TypeSystem.getInstance().getDataType(ClassType.class, i.dataType().getName());
+ clsType.validateId((Id) val);
+ cVal = val;
+ } else {
+ cVal = i.dataType().convert(val, i.multiplicity);
+ }
+ if (cVal == null) {
+ nullFlags[nullPos] = true;
+ return;
+ }
+ nullFlags[nullPos] = false;
+ if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
+ bools[pos] = ((Boolean) cVal).booleanValue();
+ } else if (i.dataType() == DataTypes.BYTE_TYPE) {
+ bytes[pos] = ((Byte) cVal).byteValue();
+ } else if (i.dataType() == DataTypes.SHORT_TYPE) {
+ shorts[pos] = ((Short) cVal).shortValue();
+ } else if (i.dataType() == DataTypes.INT_TYPE) {
+ ints[pos] = ((Integer) cVal).intValue();
+ } else if (i.dataType() == DataTypes.LONG_TYPE) {
+ longs[pos] = ((Long) cVal).longValue();
+ } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
+ floats[pos] = ((Float) cVal).floatValue();
+ } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
+ doubles[pos] = ((Double) cVal).doubleValue();
+ } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ bigIntegers[pos] = (BigInteger) cVal;
+ } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ bigDecimals[pos] = (BigDecimal) cVal;
+ } else if (i.dataType() == DataTypes.DATE_TYPE) {
+ dates[pos] = (Date) cVal;
+ } else if (i.dataType() == DataTypes.STRING_TYPE) {
+ strings[pos] = (String) cVal;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
+ ints[pos] = ((EnumValue) cVal).ordinal;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ arrays[pos] = (ImmutableList) cVal;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ maps[pos] = (ImmutableMap) cVal;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+ i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ structs[pos] = (StructInstance) cVal;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ if (cVal instanceof Id) {
+ ids[pos] = (Id) cVal;
+ } else {
+ referenceables[pos] = (ReferenceableInstance) cVal;
+ }
+ } else {
+ throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
+ }
+ }
+
+ public Object get(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return null;
+ }
+
+ if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
+ return bools[pos];
+ } else if (i.dataType() == DataTypes.BYTE_TYPE) {
+ return bytes[pos];
+ } else if (i.dataType() == DataTypes.SHORT_TYPE) {
+ return shorts[pos];
+ } else if (i.dataType() == DataTypes.INT_TYPE) {
+ return ints[pos];
+ } else if (i.dataType() == DataTypes.LONG_TYPE) {
+ return longs[pos];
+ } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
+ return floats[pos];
+ } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
+ return doubles[pos];
+ } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ return bigIntegers[pos];
+ } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ return bigDecimals[pos];
+ } else if (i.dataType() == DataTypes.DATE_TYPE) {
+ return dates[pos];
+ } else if (i.dataType() == DataTypes.STRING_TYPE) {
+ return strings[pos];
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
+ return ((EnumType) i.dataType()).fromOrdinal(ints[pos]);
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ return arrays[pos];
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ return maps[pos];
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
+ i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ return structs[pos];
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ if (ids[pos] != null) {
+ return ids[pos];
+ } else {
+ return referenceables[pos];
+ }
+ } else {
+ throw new MetadataException(String.format("Unknown datatype %s", i.dataType()));
+ }
+ }
+
+ public void setNull(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+ nullFlags[nullPos] = true;
+ }
+
+ /*
+ * Use only for json serialization
+ * @nonpublic
+ */
+ @Override
+ public Map<String, Object> getValuesMap() throws MetadataException {
+
+ Map<String,Object> m = new HashMap<>();
+ for (String attr : fieldMapping.fields.keySet()) {
+ m.put(attr, get(attr));
+ }
+ return m;
+ }
+
+ public boolean getBoolean(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BOOLEAN_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.BOOLEAN_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.BOOLEAN_TYPE.nullValue();
+ }
+
+ return bools[pos];
+ }
+
+ public byte getByte(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BYTE_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.BYTE_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.BYTE_TYPE.nullValue();
+ }
+
+ return bytes[pos];
+ }
+
+ public short getShort(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.SHORT_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.SHORT_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.SHORT_TYPE.nullValue();
+ }
+
+ return shorts[pos];
+ }
+
+ public int getInt(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+
+ if (i.dataType() != DataTypes.INT_TYPE && !(i.dataType() instanceof EnumType)) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.INT_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.INT_TYPE.nullValue();
+ }
+
+ return ints[pos];
+ }
+
+ public long getLong(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.LONG_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.LONG_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.LONG_TYPE.nullValue();
+ }
+
+ return longs[pos];
+ }
+
+ public float getFloat(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.FLOAT_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.FLOAT_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.FLOAT_TYPE.nullValue();
+ }
+
+ return floats[pos];
+ }
+
+ public double getDouble(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.DOUBLE_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.DOUBLE_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.DOUBLE_TYPE.nullValue();
+ }
+
+ return doubles[pos];
+ }
+
+ public BigInteger getBigInt(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BIGINTEGER_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.BIGINTEGER_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.BIGINTEGER_TYPE.nullValue();
+ }
+
+ return bigIntegers[pos];
+ }
+
+ public BigDecimal getBigDecimal(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BIGDECIMAL_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.BIGDECIMAL_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.BIGDECIMAL_TYPE.nullValue();
+ }
+
+ return bigDecimals[pos];
+ }
+
+ public Date getDate(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.DATE_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.DATE_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.DATE_TYPE.nullValue();
+ }
+
+ return dates[pos];
+ }
+
+ public String getString(String attrName) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.STRING_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic get method",
+ attrName, getTypeName(), DataTypes.STRING_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ if (nullFlags[nullPos]) {
+ return DataTypes.STRING_TYPE.nullValue();
+ }
+
+ return strings[pos];
+ }
+
+ public void setBoolean(String attrName, boolean val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BOOLEAN_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.BOOLEAN_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = false;
+ bools[pos] = val;
+ }
+
+ public void setByte(String attrName, byte val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BYTE_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.BYTE_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = false;
+ bytes[pos] = val;
+ }
+
+ public void setShort(String attrName, short val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.SHORT_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.SHORT_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = false;
+ shorts[pos] = val;
+ }
+
+ public void setInt(String attrName, int val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.INT_TYPE && !(i.dataType() instanceof EnumType)) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.INT_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = false;
+ ints[pos] = val;
+ }
+
+ public void setLong(String attrName, long val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.LONG_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.LONG_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = false;
+ longs[pos] = val;
+ }
+
+ public void setFloat(String attrName, float val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.FLOAT_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.FLOAT_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = false;
+ floats[pos] = val;
+ }
+
+ public void setDouble(String attrName, double val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.DOUBLE_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.DOUBLE_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = false;
+ doubles[pos] = val;
+ }
+
+ public void setBigInt(String attrName, BigInteger val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BIGINTEGER_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.BIGINTEGER_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = val == null;
+ bigIntegers[pos] = val;
+ }
+
+ public void setBigDecimal(String attrName, BigDecimal val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.BIGDECIMAL_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.BIGDECIMAL_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = val == null;
+ bigDecimals[pos] = val;
+ }
+
+ public void setDate(String attrName, Date val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.DATE_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.DATE_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = val == null;
+ dates[pos] = val;
+ }
+
+ public void setString(String attrName, String val) throws MetadataException {
+ AttributeInfo i = fieldMapping.fields.get(attrName);
+ if (i == null) {
+ throw new MetadataException(
+ String.format("Unknown field %s for Struct %s", attrName, getTypeName()));
+ }
+
+ if (i.dataType() != DataTypes.STRING_TYPE) {
+ throw new MetadataException(
+ String.format("Field %s for Struct %s is not a %s, call generic set method",
+ attrName, getTypeName(), DataTypes.STRING_TYPE.getName()));
+ }
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+ int nullPos = fieldMapping.fieldNullPos.get(attrName);
+
+ nullFlags[nullPos] = val == null;
+ strings[pos] = val;
+ }
+
+ public void output(IStruct s, Appendable buf, String prefix) throws MetadataException {
+ TypeUtils.outputVal("{", buf, prefix);
+ if (s == null) {
+ TypeUtils.outputVal("<null>\n", buf, "");
+ return;
+ }
+ TypeUtils.outputVal("\n", buf, "");
+ String fieldPrefix = prefix + "\t";
+ for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
+ String attrName = e.getKey();
+ AttributeInfo i = e.getValue();
+ Object aVal = s.get(attrName);
+ TypeUtils.outputVal(attrName + " : ", buf, fieldPrefix);
+ i.dataType().output(aVal, buf, "");
+ TypeUtils.outputVal("\n", buf, "");
+ }
+ TypeUtils.outputVal("\n}\n", buf, "");
+ }
+
+ @Override
+ public String toString() {
+ try {
+ StringBuilder buf = new StringBuilder();
+ String prefix = "";
+
+ fieldMapping.output(this, buf, prefix);
+ return buf.toString();
+
+ } catch (MetadataException me) {
+ throw new RuntimeException(me);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
new file mode 100755
index 0000000..a3287fc
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.MetadataException;
+
+abstract class AbstractDataType<T> implements IDataType<T> {
+
+ protected T convertNull(Multiplicity m) throws MetadataException {
+ if (!m.nullAllowed()) {
+ throw new ValueConversionException.NullConversionException(m);
+ }
+ return null;
+ }
+
+ @Override
+ public void output(T val, Appendable buf, String prefix) throws MetadataException {
+ TypeUtils.outputVal(val == null ? "<null>" : val.toString(), buf, prefix);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
new file mode 100755
index 0000000..e18305f
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.ParamChecker;
+
+public final class AttributeDefinition {
+
+ public final String name;
+ public final String dataTypeName;
+ public final Multiplicity multiplicity;
+ public final boolean isComposite;
+ public final boolean isUnique;
+ public final boolean isIndexable;
+
+ /**
+ * If this is a reference attribute, then the name of the attribute on the Class
+ * that this refers to.
+ */
+ public final String reverseAttributeName;
+
+ public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity,
+ boolean isComposite, String reverseAttributeName) {
+ this(name, dataTypeName, multiplicity, isComposite, false, true, reverseAttributeName);
+
+ }
+
+ public AttributeDefinition(String name, String dataTypeName,
+ Multiplicity multiplicity, boolean isComposite, boolean isUnique,
+ boolean isIndexable, String reverseAttributeName) {
+ this.name = ParamChecker.notEmpty(name, "Attribute name");
+ this.dataTypeName = ParamChecker.notEmpty(dataTypeName, "Attribute type");
+ this.multiplicity = multiplicity;
+ this.isComposite = isComposite;
+ this.isUnique = isUnique;
+ this.isIndexable = isIndexable;
+ this.reverseAttributeName = ParamChecker.notEmptyIfNotNull(reverseAttributeName, "Reverse attribute name");
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ AttributeDefinition that = (AttributeDefinition) o;
+
+ if (isComposite != that.isComposite) return false;
+ if (isUnique != that.isUnique) return false;
+ if (isIndexable != that.isIndexable) return false;
+ if (!dataTypeName.equals(that.dataTypeName)) return false;
+ if (!multiplicity.equals(that.multiplicity)) return false;
+ if (!name.equals(that.name)) return false;
+ if (reverseAttributeName != null
+ ? !reverseAttributeName.equals(that.reverseAttributeName)
+ : that.reverseAttributeName != null)
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name.hashCode();
+ result = 31 * result + dataTypeName.hashCode();
+ result = 31 * result + multiplicity.hashCode();
+ result = 31 * result + (isComposite ? 1 : 0);
+ result = 31 * result + (isUnique ? 1 : 0);
+ result = 31 * result + (isIndexable ? 1 : 0);
+ result = 31 * result + (reverseAttributeName != null ? reverseAttributeName.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
new file mode 100755
index 0000000..8761013
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.MetadataException;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.Map;
+
+public class AttributeInfo {
+ public final String name;
+ public final Multiplicity multiplicity;
+ public final boolean isComposite;
+ public final boolean isUnique;
+ public final boolean isIndexable;
+ /**
+ * If this is a reference attribute, then the name of the attribute on the Class
+ * that this refers to.
+ */
+ public final String reverseAttributeName;
+ private IDataType dataType;
+
+ AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws MetadataException {
+ this.name = def.name;
+ this.dataType = (tempTypes != null && tempTypes.containsKey(def.dataTypeName)) ?
+ tempTypes.get(def.dataTypeName) : t.getDataType(IDataType.class, def.dataTypeName);
+ this.multiplicity = def.multiplicity;
+ this.isComposite = def.isComposite;
+ this.isUnique = def.isUnique;
+ this.isIndexable = def.isIndexable;
+ this.reverseAttributeName = def.reverseAttributeName;
+ }
+
+ public IDataType dataType() {
+ return dataType;
+ }
+
+ void setDataType(IDataType dT) {
+ dataType = dT;
+ }
+
+ @Override
+ public String toString() {
+ return "AttributeInfo{" +
+ "name='" + name + '\'' +
+ ", dataType=" + dataType +
+ ", multiplicity=" + multiplicity +
+ ", isComposite=" + isComposite +
+ ", isUnique=" + isUnique +
+ ", isIndexable=" + isIndexable +
+ ", reverseAttributeName='" + reverseAttributeName + '\'' +
+ '}';
+ }
+
+ public String toJson() throws JSONException {
+ JSONObject json = new JSONObject();
+ json.put("name", name);
+ json.put("multiplicity", multiplicity.toJson());
+ json.put("isComposite", isComposite);
+ json.put("isUnique", isUnique);
+ json.put("isIndexable", isIndexable);
+ json.put("dataType", dataType.getName());
+ json.put("reverseAttributeName", reverseAttributeName);
+ return json.toString();
+ }
+
+ public static AttributeDefinition fromJson(String jsonStr) throws JSONException {
+ JSONObject json = new JSONObject(jsonStr);
+ String reverseAttr = null;
+ if (json.has("reverseAttributeName")) {
+ reverseAttr = json.getString("reverseAttributeName");
+ }
+ return new AttributeDefinition(json.getString("name"), json.getString("dataType"),
+ Multiplicity.fromJson(json.getString("multiplicity")), json.getBoolean("isComposite"),
+ json.getBoolean("isUnique"), json.getBoolean("isIndexable"), reverseAttr);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
new file mode 100755
index 0000000..f7b2721
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.StructInstance;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class ClassType extends HierarchicalType<ClassType, IReferenceableInstance>
+ implements IConstructableType<IReferenceableInstance, ITypedReferenceableInstance> {
+
+ public static final String TRAIT_NAME_SEP = "::";
+
+ public final Map<AttributeInfo, List<String>> infoToNameMap;
+
+ /**
+ * Used when creating a ClassType, to support recursive Structs.
+ */
+ ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, int numFields) {
+ super(typeSystem, ClassType.class, name, superTypes, numFields);
+ infoToNameMap = null;
+ }
+
+ ClassType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes,
+ AttributeInfo... fields)
+ throws MetadataException {
+ super(typeSystem, ClassType.class, name, superTypes, fields);
+ infoToNameMap = TypeUtils.buildAttrInfoToNameMap(fieldMapping);
+ }
+
+ @Override
+ public DataTypes.TypeCategory getTypeCategory() {
+ return DataTypes.TypeCategory.CLASS;
+ }
+
+ public void validateId(Id id) throws MetadataException {
+ if (id != null) {
+ ClassType cType = typeSystem.getDataType(ClassType.class, id.className);
+ if (isSubType(cType.getName())) {
+ return;
+ }
+ throw new MetadataException(
+ String.format("Id %s is not valid for class %s", id, getName()));
+ }
+ }
+
+ protected Id getId(Object val) throws MetadataException {
+ if (val instanceof Referenceable) {
+ return ((Referenceable) val).getId();
+ }
+ throw new MetadataException(String.format("Cannot get id from class %s", val.getClass()));
+ }
+
+ @Override
+ public ITypedReferenceableInstance convert(Object val, Multiplicity m)
+ throws MetadataException {
+
+ if (val != null) {
+ if (val instanceof ITypedReferenceableInstance) {
+ ITypedReferenceableInstance tr = (ITypedReferenceableInstance) val;
+ if (!tr.getTypeName().equals(getName())) {
+ /*
+ * If val is a subType instance; invoke convert on it.
+ */
+ ClassType valType = typeSystem.getDataType(superTypeClass, tr.getTypeName());
+ if (valType.superTypePaths.containsKey(name)) {
+ return valType.convert(val, m);
+ }
+ throw new ValueConversionException(this, val);
+ }
+ return tr;
+ } else if (val instanceof Struct) {
+ Struct s = (Struct) val;
+ Referenceable r = null;
+ Id id = null;
+
+ if (!s.typeName.equals(getName())) {
+ /*
+ * If val is a subType instance; invoke convert on it.
+ */
+ ClassType valType = typeSystem.getDataType(superTypeClass, s.typeName);
+ if (valType.superTypePaths.containsKey(name)) {
+ return valType.convert(s, m);
+ }
+ throw new ValueConversionException(this, val);
+ }
+
+ if (val instanceof Referenceable) {
+ r = (Referenceable) val;
+ id = r.getId();
+ }
+
+ ITypedReferenceableInstance tr = r != null ?
+ createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0]))
+ : createInstance(id);
+
+ if (id != null && id.isAssigned()) {
+ return tr;
+ }
+
+ for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
+ String attrKey = e.getKey();
+ AttributeInfo i = e.getValue();
+ Object aVal = s.get(attrKey);
+ if (aVal != null &&
+ i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ if (!i.isComposite) {
+ aVal = ((IReferenceableInstance) aVal).getId();
+ }
+ }
+
+ try {
+ tr.set(attrKey, aVal);
+ } catch (ValueConversionException ve) {
+ throw new ValueConversionException(this, val, ve);
+ }
+ }
+
+ return tr;
+ } else if (val instanceof ReferenceableInstance) {
+ validateId(((ReferenceableInstance) val).getId());
+ return (ReferenceableInstance) val;
+ } else {
+ throw new ValueConversionException(this, val);
+ }
+ }
+ if (!m.nullAllowed()) {
+ throw new ValueConversionException.NullConversionException(m);
+ }
+ return null;
+ }
+
+ @Override
+ public ITypedReferenceableInstance createInstance() throws MetadataException {
+ return createInstance((String[])null);
+ }
+
+ public ITypedReferenceableInstance createInstance(String... traitNames)
+ throws MetadataException {
+ return createInstance(null, traitNames);
+ }
+
+ public ITypedReferenceableInstance createInstance(Id id, String... traitNames)
+ throws MetadataException {
+ return createInstanceWithTraits(id, null, traitNames);
+ }
+
+ public ITypedReferenceableInstance createInstanceWithTraits(Id id, Referenceable r,
+ String... traitNames)
+ throws MetadataException {
+
+ ImmutableMap.Builder<String, ITypedStruct> b
+ = new ImmutableBiMap.Builder<String, ITypedStruct>();
+ if (traitNames != null) {
+ for (String t : traitNames) {
+ TraitType tType = typeSystem.getDataType(TraitType.class, t);
+ IStruct iTraitObject = r == null ? null : r.getTrait(t);
+ ITypedStruct trait = iTraitObject == null ? tType.createInstance() :
+ tType.convert(iTraitObject, Multiplicity.REQUIRED);
+ b.put(t, trait);
+ }
+ }
+
+ return new ReferenceableInstance(id == null ? new Id(getName()) : id,
+ getName(),
+ fieldMapping,
+ new boolean[fieldMapping.fields.size()],
+ fieldMapping.numBools == 0 ? null : new boolean[fieldMapping.numBools],
+ fieldMapping.numBytes == 0 ? null : new byte[fieldMapping.numBytes],
+ fieldMapping.numShorts == 0 ? null : new short[fieldMapping.numShorts],
+ fieldMapping.numInts == 0 ? null : new int[fieldMapping.numInts],
+ fieldMapping.numLongs == 0 ? null : new long[fieldMapping.numLongs],
+ fieldMapping.numFloats == 0 ? null : new float[fieldMapping.numFloats],
+ fieldMapping.numDoubles == 0 ? null : new double[fieldMapping.numDoubles],
+ fieldMapping.numBigDecimals == 0 ? null
+ : new BigDecimal[fieldMapping.numBigDecimals],
+ fieldMapping.numBigInts == 0 ? null : new BigInteger[fieldMapping.numBigInts],
+ fieldMapping.numDates == 0 ? null : new Date[fieldMapping.numDates],
+ fieldMapping.numStrings == 0 ? null : new String[fieldMapping.numStrings],
+ fieldMapping.numArrays == 0 ? null : new ImmutableList[fieldMapping.numArrays],
+ fieldMapping.numMaps == 0 ? null : new ImmutableMap[fieldMapping.numMaps],
+ fieldMapping.numStructs == 0 ? null : new StructInstance[fieldMapping.numStructs],
+ fieldMapping.numReferenceables == 0 ? null
+ : new ReferenceableInstance[fieldMapping.numReferenceables],
+ fieldMapping.numReferenceables == 0 ? null : new Id[fieldMapping.numReferenceables],
+ b.build());
+ }
+
+ @Override
+ public void output(IReferenceableInstance s, Appendable buf, String prefix)
+ throws MetadataException {
+ fieldMapping.output(s, buf, prefix);
+ }
+
+ @Override
+ public List<String> getNames(AttributeInfo info) {
+ return infoToNameMap.get(info);
+ }
+}
\ No newline at end of file
[53/58] [abbrv] incubator-atlas git commit: HDPDGI-82:Vishal:Load d3
js from lib folder instead of cdn
Posted by ve...@apache.org.
HDPDGI-82:Vishal:Load d3 js from lib folder instead of cdn
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/6c896a7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/6c896a7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/6c896a7a
Branch: refs/heads/master
Commit: 6c896a7af360c21dec207fc0cad92a715264f116
Parents: 63c86bf
Author: Vishal Kadam <vi...@gmail.com>
Authored: Tue Jun 16 00:39:10 2015 -0400
Committer: Vishal Kadam <vi...@gmail.com>
Committed: Tue Jun 16 00:39:10 2015 -0400
----------------------------------------------------------------------
dashboard/v2/public/index.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6c896a7a/dashboard/v2/public/index.html
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/index.html b/dashboard/v2/public/index.html
index 4a2e824..90356f3 100644
--- a/dashboard/v2/public/index.html
+++ b/dashboard/v2/public/index.html
@@ -64,7 +64,7 @@
<script src="lib/angular-ui-router/release/angular-ui-router.js"></script>
<script src="lib/angular-ui-utils/ui-utils.js"></script>
<script src="lib/lodash/lodash.js"></script>
-<script src="http://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
+<script src="/lib/d3/d3.js"></script>
<script src="js/app.min.js"></script>
</body>
[07/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java b/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java
new file mode 100755
index 0000000..1a512a1
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+
+import java.io.Console;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
+
+/**
+ * A utility class for generating a credential provider containing the entries required for supporting the SSL implementation
+ * of the DGC server.
+ */
+public class CredentialProviderUtility {
+ private static final String[] KEYS = new String[] {KEYSTORE_PASSWORD_KEY,
+ TRUSTSTORE_PASSWORD_KEY, SERVER_CERT_PASSWORD_KEY};
+
+ public static abstract class TextDevice {
+ public abstract void printf(String fmt, Object... params);
+
+ public abstract String readLine(String fmt, Object ... args);
+
+ public abstract char[] readPassword(String fmt, Object ... args);
+
+ }
+
+ private static TextDevice DEFAULT_TEXT_DEVICE = new TextDevice() {
+ Console console = System.console();
+
+ @Override
+ public void printf(String fmt, Object... params) {
+ console.printf(fmt, params);
+ }
+
+ @Override
+ public String readLine(String fmt, Object ... args) {
+ return console.readLine(fmt, args);
+ }
+
+ @Override
+ public char[] readPassword(String fmt, Object ... args) {
+ return console.readPassword(fmt, args);
+ }
+ };
+
+ public static TextDevice textDevice = DEFAULT_TEXT_DEVICE;
+
+ public static void main(String[] args) throws IOException {
+ // prompt for the provider name
+ CredentialProvider provider = getCredentialProvider(textDevice);
+
+ char[] cred;
+ for (String key : KEYS) {
+ cred = getPassword(textDevice, key);
+ // create a credential entry and store it
+ boolean overwrite = true;
+ if (provider.getCredentialEntry(key) != null) {
+ String choice = textDevice.readLine("Entry for %s already exists. Overwrite? (y/n) [y]:", key);
+ overwrite = StringUtils.isEmpty(choice) || choice.equalsIgnoreCase("y");
+ if (overwrite) {
+ provider.deleteCredentialEntry(key);
+ provider.flush();
+ provider.createCredentialEntry(key, cred);
+ provider.flush();
+ textDevice.printf("Entry for %s was overwritten with the new value.\n", key);
+ } else {
+ textDevice.printf("Entry for %s was not overwritten.\n", key);
+ }
+ } else {
+ provider.createCredentialEntry(key, cred);
+ provider.flush();
+ }
+ }
+ }
+
+ /**
+ * Retrieves a password from the command line.
+ * @param textDevice the system console.
+ * @param key the password key/alias.
+ * @return the password.
+ */
+ private static char[] getPassword(TextDevice textDevice, String key) {
+ boolean noMatch;
+ char[] cred = new char[0];
+ char[] passwd1;
+ char[] passwd2;
+ do {
+ passwd1 = textDevice.readPassword("Please enter the password value for %s:", key);
+ passwd2 = textDevice.readPassword("Please enter the password value for %s again:", key);
+ noMatch = !Arrays.equals(passwd1, passwd2);
+ if (noMatch) {
+ if (passwd1 != null) Arrays.fill(passwd1, ' ');
+ textDevice.printf("Password entries don't match. Please try again.\n");
+ } else {
+ if (passwd1.length == 0) {
+ textDevice.printf("An empty password is not valid. Please try again.\n");
+ noMatch = true;
+ } else {
+ cred = passwd1;
+ }
+ }
+ if (passwd2 != null) Arrays.fill(passwd2, ' ');
+ } while (noMatch);
+ return cred;
+ }
+
+ /**\
+ * Returns a credential provider for the entered JKS path.
+ * @param textDevice the system console.
+ * @return the Credential provider
+ * @throws IOException
+ */
+ private static CredentialProvider getCredentialProvider(TextDevice textDevice) throws IOException {
+ String providerPath = textDevice.readLine("Please enter the full path to the credential provider:");
+ File file = new File(providerPath);
+ if (file.exists()) {
+ textDevice.printf("%s already exists. You will need to specify whether existing entries should be overwritten " +
+ "(default is 'yes')\n", providerPath);
+ }
+ String providerURI = JavaKeyStoreProvider.SCHEME_NAME + "://file" + providerPath;
+ Configuration conf = new Configuration(false);
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerURI);
+ return CredentialProviderFactory.getProviders(conf).get(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java b/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java
new file mode 100755
index 0000000..bdf84a6
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.errors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Exception mapper for Jersey.
+ * @param <E>
+ */
+public class LoggingExceptionMapper<E extends Throwable> implements ExceptionMapper<E> {
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoggingExceptionMapper.class);
+
+ @Override
+ public Response toResponse(E exception) {
+ if (exception instanceof WebApplicationException) {
+ return ((WebApplicationException) exception).getResponse();
+ }
+
+ final long id = ThreadLocalRandom.current().nextLong();
+ logException(id, exception);
+ return Response.serverError()
+ .entity(formatErrorMessage(id, exception))
+ .build();
+ }
+
+ @SuppressWarnings("UnusedParameters")
+ protected String formatErrorMessage(long id, E exception) {
+ return String.format(
+ "There was an error processing your request. It has been logged (ID %016x).", id);
+ }
+
+ protected void logException(long id, E exception) {
+ LOGGER.error(formatLogMessage(id, exception), exception);
+ }
+
+ @SuppressWarnings("UnusedParameters")
+ protected String formatLogMessage(long id, Throwable exception) {
+ return String.format("Error handling a request: %016x", id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
new file mode 100755
index 0000000..0ebf22d
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.filters;
+
+import com.google.inject.Singleton;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.web.util.DateTimeHelper;
+import org.apache.atlas.web.util.Servlets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.Date;
+import java.util.UUID;
+
+/**
+ * This records audit information as part of the filter after processing the request
+ * and also introduces a UUID into request and response for tracing requests in logs.
+ */
+@Singleton
+public class AuditFilter implements Filter {
+
+ private static final Logger AUDIT_LOG = LoggerFactory.getLogger("AUDIT");
+ private static final Logger LOG = LoggerFactory.getLogger(AuditFilter.class);
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ LOG.info("AuditFilter initialization started");
+ }
+
+ @Override
+ public void doFilter(ServletRequest request,
+ ServletResponse response,
+ FilterChain filterChain) throws IOException, ServletException {
+ final String requestTimeISO9601 = DateTimeHelper.formatDateUTC(new Date());
+ final HttpServletRequest httpRequest = (HttpServletRequest) request;
+ final String requestId = UUID.randomUUID().toString();
+ final Thread currentThread = Thread.currentThread();
+ final String oldName = currentThread.getName();
+
+ try {
+ currentThread.setName(formatName(oldName, requestId));
+ recordAudit(httpRequest, requestTimeISO9601);
+ filterChain.doFilter(request, response);
+ } finally {
+ // put the request id into the response so users can trace logs for this request
+ ((HttpServletResponse) response).setHeader(MetadataServiceClient.REQUEST_ID, requestId);
+ currentThread.setName(oldName);
+ }
+ }
+
+ private String formatName(String oldName, String requestId) {
+ return oldName + " - " + requestId;
+ }
+
+ private void recordAudit(HttpServletRequest httpRequest, String whenISO9601) {
+ final String who = getUserFromRequest(httpRequest);
+ final String fromHost = httpRequest.getRemoteHost();
+ final String fromAddress = httpRequest.getRemoteAddr();
+ final String whatURL = Servlets.getRequestURL(httpRequest);
+ final String whatAddrs = httpRequest.getLocalAddr();
+
+ LOG.debug("Audit: {}/{} performed request {} ({}) at time {}",
+ who, fromAddress, whatURL, whatAddrs, whenISO9601);
+ audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
+ }
+
+ private String getUserFromRequest(HttpServletRequest httpRequest) {
+ // look for the user in the request
+ final String userFromRequest = Servlets.getUserFromRequest(httpRequest);
+ return userFromRequest == null ? "UNKNOWN" : userFromRequest;
+ }
+
+ private void audit(String who, String fromAddress, String fromHost, String whatURL,
+ String whatAddrs, String whenISO9601) {
+ AUDIT_LOG.info("Audit: {}/{}-{} performed request {} ({}) at time {}",
+ who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
+ }
+
+ @Override
+ public void destroy() {
+ // do nothing
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java
new file mode 100644
index 0000000..5e9e453
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.filters;
+
+import com.google.inject.Singleton;
+import org.apache.atlas.PropertiesUtil;
+import org.apache.atlas.security.SecurityProperties;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.Properties;
+
+/**
+ * This enforces authentication as part of the filter before processing the request.
+ * todo: Subclass of {@link org.apache.hadoop.security.authentication.server.AuthenticationFilter}.
+ */
+@Singleton
+public class MetadataAuthenticationFilter extends AuthenticationFilter {
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataAuthenticationFilter.class);
+ static final String PREFIX = "atlas.http.authentication.";
+
+ @Override
+ protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException {
+ PropertiesConfiguration configuration;
+ try {
+ configuration = PropertiesUtil.getApplicationProperties();
+ } catch (Exception e) {
+ throw new ServletException(e);
+ }
+
+ Properties config = new Properties();
+
+ config.put(AuthenticationFilter.COOKIE_PATH, "/");
+
+ // add any config passed in as init parameters
+ Enumeration<String> enumeration = filterConfig.getInitParameterNames();
+ while (enumeration.hasMoreElements()) {
+ String name = enumeration.nextElement();
+ config.put(name, filterConfig.getInitParameter(name));
+ }
+ // transfer application.properties config items starting with defined prefix
+ Iterator<String> itor = configuration.getKeys();
+ while (itor.hasNext()) {
+ String name = itor.next();
+ if (name.startsWith(PREFIX)) {
+ String value = configuration.getString(name);
+ name = name.substring(PREFIX.length());
+ config.put(name, value);
+ }
+ }
+
+ //Resolve _HOST into bind address
+ String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS);
+ if (bindAddress == null) {
+ LOG.info("No host name configured. Defaulting to local host name.");
+ try {
+ bindAddress = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ throw new ServletException("Unable to obtain host name", e);
+ }
+ }
+ String principal = config.getProperty(KerberosAuthenticationHandler.PRINCIPAL);
+ if (principal != null) {
+ try {
+ principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
+ } catch (IOException ex) {
+ throw new RuntimeException("Could not resolve Kerberos principal name: " + ex.toString(), ex);
+ }
+ config.put(KerberosAuthenticationHandler.PRINCIPAL, principal);
+ }
+
+ LOG.info("AuthenticationFilterConfig: {}", config);
+
+ return config;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java
new file mode 100755
index 0000000..5fb5b6b
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.listeners;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.sun.jersey.api.core.PackagesResourceConfig;
+import com.sun.jersey.guice.JerseyServletModule;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.PropertiesUtil;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.web.filters.AuditFilter;
+import org.apache.atlas.web.filters.MetadataAuthenticationFilter;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletContextEvent;
+import java.util.HashMap;
+import java.util.Map;
+
+public class GuiceServletConfig extends GuiceServletContextListener {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GuiceServletConfig.class);
+
+ private static final String GUICE_CTX_PARAM = "guice.packages";
+ static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled";
+ private Injector injector;
+
+ @Override
+ protected Injector getInjector() {
+ LOG.info("Loading Guice modules");
+ /*
+ * More information on this can be found here:
+ * https://jersey.java.net/nonav/apidocs/1
+ * .11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary
+ * .html
+ */
+ if (injector == null) {
+ injector = Guice.createInjector(
+ new RepositoryMetadataModule(),
+ new JerseyServletModule() {
+ @Override
+ protected void configureServlets() {
+ filter("/*").through(AuditFilter.class);
+ try {
+ configureAuthenticationFilter();
+ } catch (ConfigurationException e) {
+ LOG.warn("Unable to add and configure authentication filter", e);
+ }
+
+ String packages = getServletContext().getInitParameter(GUICE_CTX_PARAM);
+
+ LOG.info("Jersey loading from packages: " + packages);
+
+ Map<String, String> params = new HashMap<>();
+ params.put(PackagesResourceConfig.PROPERTY_PACKAGES, packages);
+ serve("/" + MetadataServiceClient.BASE_URI + "*").with(GuiceContainer.class, params);
+ }
+
+ private void configureAuthenticationFilter() throws ConfigurationException {
+ try {
+ PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
+ if (Boolean.valueOf(configuration.getString(HTTP_AUTHENTICATION_ENABLED))) {
+ filter("/*").through(MetadataAuthenticationFilter.class);
+ }
+ } catch (MetadataException e) {
+ LOG.warn("Error loading configuration and initializing authentication filter", e);
+ }
+ }
+ });
+
+ LOG.info("Guice modules loaded");
+ }
+
+ return injector;
+ }
+
+ @Override
+ public void contextInitialized(ServletContextEvent servletContextEvent) {
+ super.contextInitialized(servletContextEvent);
+
+ // perform login operations
+ LoginProcessor loginProcessor = new LoginProcessor();
+ loginProcessor.login();
+ }
+
+ @Override
+ public void contextDestroyed(ServletContextEvent servletContextEvent) {
+ super.contextDestroyed(servletContextEvent);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java b/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java
new file mode 100644
index 0000000..2c6d98e
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.listeners;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.PropertiesUtil;
+import org.apache.atlas.security.SecurityProperties;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * A class capable of performing a simple or kerberos login.
+ */
+public class LoginProcessor {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(LoginProcessor.class);
+ public static final String METADATA_AUTHENTICATION_PREFIX = "atlas.authentication.";
+ public static final String AUTHENTICATION_METHOD = METADATA_AUTHENTICATION_PREFIX + "method";
+ public static final String AUTHENTICATION_PRINCIPAL = METADATA_AUTHENTICATION_PREFIX + "principal";
+ public static final String AUTHENTICATION_KEYTAB = METADATA_AUTHENTICATION_PREFIX + "keytab";
+
+ /**
+ * Perform a SIMPLE login based on established OS identity or a kerberos based login using the configured
+ * principal and keytab (via application.properties).
+ */
+ public void login() {
+ // first, let's see if we're running in a hadoop cluster and have the env configured
+ boolean isHadoopCluster = isHadoopCluster();
+ Configuration hadoopConfig = isHadoopCluster ? getHadoopConfiguration() : new Configuration(false);
+ PropertiesConfiguration configuration = null;
+ try {
+ configuration = getPropertiesConfiguration();
+ } catch (ConfigurationException e) {
+ LOG.warn("Error reading application configuration", e);
+ }
+ if (!isHadoopCluster) {
+ // need to read the configured authentication choice and create the UGI configuration
+ setupHadoopConfiguration(hadoopConfig, configuration);
+ }
+ doServiceLogin(hadoopConfig, configuration);
+ }
+
+ protected void doServiceLogin(Configuration hadoopConfig, PropertiesConfiguration configuration) {
+ UserGroupInformation.setConfiguration(hadoopConfig);
+
+ UserGroupInformation ugi = null;
+ UserGroupInformation.AuthenticationMethod authenticationMethod =
+ SecurityUtil.getAuthenticationMethod(hadoopConfig);
+ try {
+ if (authenticationMethod == UserGroupInformation.AuthenticationMethod.SIMPLE) {
+ UserGroupInformation.loginUserFromSubject(null);
+ } else if (authenticationMethod == UserGroupInformation.AuthenticationMethod.KERBEROS) {
+ String bindAddress = getHostname(configuration);
+ UserGroupInformation.loginUserFromKeytab(
+ getServerPrincipal(configuration.getString(AUTHENTICATION_PRINCIPAL), bindAddress),
+ configuration.getString(AUTHENTICATION_KEYTAB));
+ }
+ LOG.info("Logged in user {}", UserGroupInformation.getLoginUser());
+ } catch (IOException e) {
+ throw new IllegalStateException(String.format("Unable to perform %s login.", authenticationMethod), e);
+ }
+ }
+
+ private String getHostname(PropertiesConfiguration configuration) {
+ String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS);
+ if (bindAddress == null) {
+ LOG.info("No host name configured. Defaulting to local host name.");
+ try {
+ bindAddress = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ return bindAddress;
+ }
+
+ protected void setupHadoopConfiguration(Configuration hadoopConfig, PropertiesConfiguration configuration) {
+ String authMethod;
+ authMethod = configuration != null ? configuration.getString(AUTHENTICATION_METHOD) : null;
+ // getString may return null, and would like to log the nature of the default setting
+ if (authMethod == null) {
+ LOG.info("No authentication method configured. Defaulting to simple authentication");
+ authMethod = "simple";
+ }
+ SecurityUtil.setAuthenticationMethod(
+ UserGroupInformation.AuthenticationMethod.valueOf(authMethod.toUpperCase()),
+ hadoopConfig);
+ }
+
+ /**
+ * Return a server (service) principal. The token "_HOST" in the principal will be replaced with the local host
+ * name (e.g. dgi/_HOST will be changed to dgi/localHostName)
+ * @param principal the input principal containing an option "_HOST" token
+ * @return the service principal.
+ * @throws IOException
+ */
+ private String getServerPrincipal(String principal, String host) throws IOException {
+ return SecurityUtil.getServerPrincipal(principal, host);
+ }
+
+ /**
+ * Returns a Hadoop configuration instance.
+ * @return the configuration.
+ */
+ protected Configuration getHadoopConfiguration() {
+ return new Configuration();
+ }
+
+ /**
+ * Returns the metadata application configuration.
+ * @return the metadata configuration.
+ * @throws ConfigurationException
+ */
+ protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
+ try {
+ return PropertiesUtil.getApplicationProperties();
+ } catch (MetadataException e) {
+ throw new ConfigurationException(e);
+ }
+ }
+
+ /**
+ * Uses a hadoop shell to discern whether a hadoop cluster is available/configured.
+ * @return true if a hadoop cluster is detected.
+ */
+ protected boolean isHadoopCluster() {
+ boolean isHadoopCluster = false;
+ try {
+ isHadoopCluster = Shell.getHadoopHome() != null;
+ } catch (IOException e) {
+ // ignore - false is default setting
+ }
+ return isHadoopCluster;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java b/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java
new file mode 100755
index 0000000..f5b555e
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.params;
+
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+/**
+ * An abstract base class from which to build Jersey parameter classes.
+ *
+ * @param <T> the type of value wrapped by the parameter
+ */
+public abstract class AbstractParam<T> {
+ private final T value;
+
+ /**
+ * Given an input value from a client, creates a parameter wrapping its parsed value.
+ *
+ * @param input an input value from a client request
+ */
+ @SuppressWarnings({"AbstractMethodCallInConstructor",
+ "OverriddenMethodCallDuringObjectConstruction"})
+ protected AbstractParam(String input) {
+ try {
+ this.value = parse(input);
+ } catch (Exception e) {
+ throw new WebApplicationException(error(input, e));
+ }
+ }
+
+ /**
+ * Given a string representation which was unable to be parsed and the exception thrown, produce
+ * a {@link javax.ws.rs.core.Response} to be sent to the client.
+ *
+ * By default, generates a {@code 400 Bad Request} with a plain text entity generated by
+ * {@link #errorMessage(String, Exception)}.
+ *
+ * @param input the raw input value
+ * @param e the exception thrown while parsing {@code input}
+ * @return the {@link javax.ws.rs.core.Response} to be sent to the client
+ */
+ protected Response error(String input, Exception e) {
+ return Response.status(getErrorStatus())
+ .entity(errorMessage(input, e))
+ .type(mediaType())
+ .build();
+ }
+
+ /**
+ * Returns the media type of the error message entity.
+ *
+ * @return the media type of the error message entity
+ */
+ protected MediaType mediaType() {
+ return MediaType.TEXT_PLAIN_TYPE;
+ }
+
+ /**
+ * Given a string representation which was unable to be parsed and the exception thrown, produce
+ * an entity to be sent to the client.
+ *
+ * @param input the raw input value
+ * @param e the exception thrown while parsing {@code input}
+ * @return the error message to be sent the client
+ */
+ protected String errorMessage(String input, Exception e) {
+ return String.format("Invalid parameter: %s (%s)", input, e.getMessage());
+ }
+
+ /**
+ * Given a string representation which was unable to be parsed, produce a {@link javax.ws.rs
+ * .core.Response.Status} for the
+ * {@link Response} to be sent to the client.
+ *
+ * @return the HTTP {@link javax.ws.rs.core.Response.Status} of the error message
+ */
+ @SuppressWarnings("MethodMayBeStatic")
+ protected Response.Status getErrorStatus() {
+ return Response.Status.BAD_REQUEST;
+ }
+
+ /**
+ * Given a string representation, parse it and return an instance of the parameter type.
+ *
+ * @param input the raw input
+ * @return {@code input}, parsed as an instance of {@code T}
+ * @throws Exception if there is an error parsing the input
+ */
+ protected abstract T parse(String input) throws Exception;
+
+ /**
+ * Returns the underlying value.
+ *
+ * @return the underlying value
+ */
+ public T get() {
+ return value;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if ((obj == null) || (getClass() != obj.getClass())) {
+ return false;
+ }
+ final AbstractParam<?> that = (AbstractParam<?>) obj;
+ return value.equals(that.value);
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return value.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java b/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java
new file mode 100755
index 0000000..d81fb6f
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.params;
+
+/**
+ * A parameter encapsulating boolean values. If the query parameter value is {@code "true"},
+ * regardless of case, the returned value is {@link Boolean#TRUE}. If the query parameter value is
+ * {@code "false"}, regardless of case, the returned value is {@link Boolean#FALSE}. All other
+ * values will return a {@code 400 Bad Request} response.
+ */
+public class BooleanParam extends AbstractParam<Boolean> {
+
+ public BooleanParam(String input) {
+ super(input);
+ }
+
+ @Override
+ protected String errorMessage(String input, Exception e) {
+ return '"' + input + "\" must be \"true\" or \"false\".";
+ }
+
+ @Override
+ protected Boolean parse(String input) throws Exception {
+ if ("true".equalsIgnoreCase(input)) {
+ return Boolean.TRUE;
+ }
+ if ("false".equalsIgnoreCase(input)) {
+ return Boolean.FALSE;
+ }
+ throw new Exception();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java b/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java
new file mode 100755
index 0000000..4b417e6
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.params;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * A parameter encapsulating date/time values. All non-parsable values will return a {@code 400 Bad
+ * Request} response. All values returned are in UTC.
+ */
+public class DateTimeParam extends AbstractParam<DateTime> {
+
+ public DateTimeParam(String input) {
+ super(input);
+ }
+
+ @Override
+ protected DateTime parse(String input) throws Exception {
+ return new DateTime(input, DateTimeZone.UTC);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
new file mode 100755
index 0000000..4ad27c8
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import org.apache.atlas.web.util.Servlets;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import javax.inject.Singleton;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+/**
+ * Jersey Resource for admin operations.
+ */
+@Path("admin")
+@Singleton
+public class AdminResource {
+
+ private Response version;
+
+ /**
+ * Fetches the thread stack dump for this application.
+ *
+ * @return json representing the thread stack dump.
+ */
+ @GET
+ @Path("stack")
+ @Produces(MediaType.TEXT_PLAIN)
+ public String getThreadDump() {
+ ThreadGroup topThreadGroup = Thread.currentThread().getThreadGroup();
+
+ while (topThreadGroup.getParent() != null) {
+ topThreadGroup = topThreadGroup.getParent();
+ }
+ Thread[] threads = new Thread[topThreadGroup.activeCount()];
+
+ int nr = topThreadGroup.enumerate(threads);
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < nr; i++) {
+ builder.append(threads[i].getName()).append("\nState: ").
+ append(threads[i].getState()).append("\n");
+ String stackTrace = StringUtils.join(threads[i].getStackTrace(), "\n");
+ builder.append(stackTrace);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Fetches the version for this application.
+ *
+ * @return json representing the version.
+ */
+ @GET
+ @Path("version")
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response getVersion() {
+ if (version == null) {
+ try {
+ PropertiesConfiguration configProperties =
+ new PropertiesConfiguration("atlas-buildinfo.properties");
+
+ JSONObject response = new JSONObject();
+ response.put("Version", configProperties.getString("build.version", "UNKNOWN"));
+ response.put("Name",
+ configProperties.getString("project.name", "apache-atlas"));
+ response.put("Description", configProperties.getString("project.description",
+ "Metadata Management and Data Governance Platform over Hadoop"));
+
+ // todo: add hadoop version?
+ // response.put("Hadoop", VersionInfo.getVersion() + "-r" + VersionInfo.getRevision());
+ version = Response.ok(response).build();
+ } catch (JSONException | ConfigurationException e) {
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
new file mode 100755
index 0000000..deedf45
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
@@ -0,0 +1,355 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.google.common.base.Preconditions;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.ParamChecker;
+import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.typesystem.types.ValueConversionException;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.core.UriInfo;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
+
+/**
+ * Entity management operations as REST API.
+ *
+ * An entity is an "instance" of a Type. Entities conform to the definition
+ * of the Type they correspond with.
+ */
+@Path("entities")
+@Singleton
+public class EntityResource {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
+ private static final String TRAIT_NAME = "traitName";
+
+ private final MetadataService metadataService;
+
+ @Context
+ UriInfo uriInfo;
+
+ /**
+ * Created by the Guice ServletModule and injected with the
+ * configured MetadataService.
+ *
+ * @param metadataService metadata service handle
+ */
+ @Inject
+ public EntityResource(MetadataService metadataService) {
+ this.metadataService = metadataService;
+ }
+
+ /**
+ * Submits an entity definition (instance) corresponding to a given type.
+ */
+ @POST
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response submit(@Context HttpServletRequest request) {
+ try {
+ final String entity = Servlets.getRequestPayload(request);
+ LOG.debug("submitting entity {} ", entity);
+
+ final String guid = metadataService.createEntity(entity);
+
+ UriBuilder ub = uriInfo.getAbsolutePathBuilder();
+ URI locationURI = ub.path(guid).build();
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.GUID, guid);
+ response.put(MetadataServiceClient.DEFINITION,
+ metadataService.getEntityDefinition(guid));
+
+ return Response.created(locationURI).entity(response).build();
+
+ } catch(ValueConversionException ve) {
+ LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
+ } catch (MetadataException | IllegalArgumentException e) {
+ LOG.error("Unable to persist entity instance", e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to persist entity instance", e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Fetch the complete definition of an entity given its GUID.
+ *
+ * @param guid GUID for the entity
+ */
+ @GET
+ @Path("{guid}")
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response getEntityDefinition(@PathParam("guid") String guid) {
+ try {
+ LOG.debug("Fetching entity definition for guid={} ", guid);
+ ParamChecker.notEmpty(guid, "guid cannot be null");
+ final String entityDefinition = metadataService.getEntityDefinition(guid);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.GUID, guid);
+
+ Response.Status status = Response.Status.NOT_FOUND;
+ if (entityDefinition != null) {
+ response.put(MetadataServiceClient.DEFINITION, entityDefinition);
+ status = Response.Status.OK;
+ } else {
+ response.put(MetadataServiceClient.ERROR, Servlets.escapeJsonString(
+ String.format("An entity with GUID={%s} does not exist", guid)));
+ }
+
+ return Response.status(status).entity(response).build();
+
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with GUID={} does not exist", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (MetadataException | IllegalArgumentException e) {
+ LOG.error("Bad GUID={}", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get instance definition for GUID {}", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Gets the list of entities for a given entity type.
+ *
+ * @param entityType name of a type which is unique
+ */
+ @GET
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response getEntityListByType(@QueryParam("type") String entityType) {
+ try {
+ Preconditions.checkNotNull(entityType, "Entity type cannot be null");
+
+ LOG.debug("Fetching entity list for type={} ", entityType);
+ final List<String> entityList = metadataService.getEntityList(entityType);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.TYPENAME, entityType);
+ response.put(MetadataServiceClient.RESULTS, new JSONArray(entityList));
+ response.put(MetadataServiceClient.COUNT, entityList.size());
+
+ return Response.ok(response).build();
+ } catch (NullPointerException e) {
+ LOG.error("Entity type cannot be null", e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (MetadataException | IllegalArgumentException e) {
+ LOG.error("Unable to get entity list for type {}", entityType, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get entity list for type {}", entityType, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Adds property to the given entity id
+ * @param guid entity id
+ * @param property property to add
+ * @param value property's value
+ * @return response payload as json
+ */
+ @PUT
+ @Path("{guid}")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response update(@PathParam("guid") String guid,
+ @QueryParam("property") String property,
+ @QueryParam("value") String value) {
+ try {
+ Preconditions.checkNotNull(property, "Entity property cannot be null");
+ Preconditions.checkNotNull(value, "Entity value cannot be null");
+
+ metadataService.updateEntity(guid, property, value);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Thread.currentThread().getName());
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with GUID={} does not exist", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (MetadataException | IllegalArgumentException e) {
+ LOG.error("Unable to add property {} to entity id {}", property, guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to add property {} to entity id {}", property, guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ // Trait management functions
+ /**
+ * Gets the list of trait names for a given entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return a list of trait names for the given entity guid
+ */
+ @GET
+ @Path("{guid}/traits")
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response getTraitNames(@PathParam("guid") String guid) {
+ try {
+ LOG.debug("Fetching trait names for entity={}", guid);
+ final List<String> traitNames = metadataService.getTraitNames(guid);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.GUID, guid);
+ response.put(MetadataServiceClient.RESULTS, new JSONArray(traitNames));
+ response.put(MetadataServiceClient.COUNT, traitNames.size());
+
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with GUID={} does not exist", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (MetadataException | IllegalArgumentException e) {
+ LOG.error("Unable to get trait names for entity {}", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get trait names for entity {}", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Adds a new trait to an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ */
+ @POST
+ @Path("{guid}/traits")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response addTrait(@Context HttpServletRequest request,
+ @PathParam("guid") String guid) {
+ try {
+ final String traitDefinition = Servlets.getRequestPayload(request);
+ LOG.debug("Adding trait={} for entity={} ", traitDefinition, guid);
+ metadataService.addTrait(guid, traitDefinition);
+
+ UriBuilder ub = uriInfo.getAbsolutePathBuilder();
+ URI locationURI = ub.path(guid).build();
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.GUID, guid);
+
+ return Response.created(locationURI).entity(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with GUID={} does not exist", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (MetadataException | IOException | IllegalArgumentException e) {
+ LOG.error("Unable to add trait for entity={}", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to add trait for entity={}", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Deletes a given trait from an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitName name of the trait
+ */
+ @DELETE
+ @Path("{guid}/traits/{traitName}")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response deleteTrait(@Context HttpServletRequest request,
+ @PathParam("guid") String guid,
+ @PathParam(TRAIT_NAME) String traitName) {
+ LOG.debug("Deleting trait={} from entity={} ", traitName, guid);
+ try {
+ metadataService.deleteTrait(guid, traitName);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.GUID, guid);
+ response.put(TRAIT_NAME, traitName);
+
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with GUID={} does not exist", guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (MetadataException | IllegalArgumentException e) {
+ LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
new file mode 100644
index 0000000..cfd055b
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.ParamChecker;
+import org.apache.atlas.discovery.DiscoveryException;
+import org.apache.atlas.discovery.LineageService;
+import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
+/**
+ * Jersey Resource for Hive Table Lineage.
+ */
+@Path("lineage/hive")
+@Singleton
+public class HiveLineageResource {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveLineageResource.class);
+
+ private final LineageService lineageService;
+
+ /**
+ * Created by the Guice ServletModule and injected with the
+ * configured LineageService.
+ *
+ * @param lineageService lineage service handle
+ */
+ @Inject
+ public HiveLineageResource(LineageService lineageService) {
+ this.lineageService = lineageService;
+ }
+
+ /**
+ * Returns the inputs graph for a given entity.
+ *
+ * @param tableName table name
+ */
+ @GET
+ @Path("table/{tableName}/inputs/graph")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response inputsGraph(@Context HttpServletRequest request,
+ @PathParam("tableName") String tableName) {
+ LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
+
+ try {
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ final String jsonResult = lineageService.getInputsGraph(tableName);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put("tableName", tableName);
+ response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
+
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("table entity not found for {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (DiscoveryException | IllegalArgumentException e) {
+ LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Returns the outputs graph for a given entity.
+ *
+ * @param tableName table name
+ */
+ @GET
+ @Path("table/{tableName}/outputs/graph")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response outputsGraph(@Context HttpServletRequest request,
+ @PathParam("tableName") String tableName) {
+ LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
+
+ try {
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ final String jsonResult = lineageService.getOutputsGraph(tableName);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put("tableName", tableName);
+ response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
+
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("table entity not found for {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (DiscoveryException | IllegalArgumentException e) {
+ LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Return the schema for the given tableName.
+ *
+ * @param tableName table name
+ */
+ @GET
+ @Path("table/{tableName}/schema")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response schema(@Context HttpServletRequest request,
+ @PathParam("tableName") String tableName) {
+ LOG.info("Fetching schema for tableName={}", tableName);
+
+ try {
+ ParamChecker.notEmpty(tableName, "table name cannot be null");
+ final String jsonResult = lineageService.getSchema(tableName);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put("tableName", tableName);
+ response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
+
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("table entity not found for {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (DiscoveryException | IllegalArgumentException e) {
+ LOG.error("Unable to get schema for table {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get schema for table {}", tableName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
new file mode 100755
index 0000000..9b0b0e8
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
@@ -0,0 +1,322 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.google.common.base.Preconditions;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.ParamChecker;
+import org.apache.atlas.discovery.DiscoveryException;
+import org.apache.atlas.discovery.DiscoveryService;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Jersey Resource for metadata operations.
+ */
+@Path("discovery")
+@Singleton
+public class MetadataDiscoveryResource {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
+ private static final String QUERY_TYPE_DSL = "dsl";
+ private static final String QUERY_TYPE_GREMLIN = "gremlin";
+ private static final String QUERY_TYPE_FULLTEXT = "full-text";
+
+ private final DiscoveryService discoveryService;
+
+ /**
+ * Created by the Guice ServletModule and injected with the
+ * configured DiscoveryService.
+ *
+ * @param discoveryService metadata service handle
+ */
+ @Inject
+ public MetadataDiscoveryResource(DiscoveryService discoveryService) {
+ this.discoveryService = discoveryService;
+ }
+
+ /**
+ * Search using a given query.
+ *
+ * @param query search query in raw gremlin or DSL format falling back to full text.
+ * @return JSON representing the type and results.
+ */
+ @GET
+ @Path("search")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response search(@QueryParam("query") String query) {
+ JSONObject response;
+ try { // fall back to dsl
+ ParamChecker.notEmpty(query, "query cannot be null");
+
+ if (query.startsWith("g.")) { // raw gremlin query
+ return searchUsingGremlinQuery(query);
+ }
+
+ final String jsonResultStr = discoveryService.searchByDSL(query);
+ response = new DSLJSONResponseBuilder().results(jsonResultStr)
+ .query(query)
+ .build();
+
+ } catch (IllegalArgumentException e) {
+ LOG.error("Unable to get entity list for empty query", e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable throwable) {
+ LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
+
+ try { //fall back to full-text
+ final String jsonResultStr = discoveryService.searchByFullText(query);
+ response = new FullTextJSonResponseBuilder().results(jsonResultStr)
+ .query(query)
+ .build();
+
+ } catch (DiscoveryException | IllegalArgumentException e) {
+ LOG.error("Unable to get entity list for query {}", query, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch(Throwable e) {
+ LOG.error("Unable to get entity list for query {}", query, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ return Response.ok(response)
+ .build();
+
+ }
+
+ /**
+ * Search using query DSL format.
+ *
+ * @param dslQuery search query in DSL format.
+ * @return JSON representing the type and results.
+ */
+ @GET
+ @Path("search/dsl")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response searchUsingQueryDSL(@QueryParam("query") String dslQuery) {
+ try {
+ ParamChecker.notEmpty(dslQuery, "dslQuery cannot be null");
+ final String jsonResultStr = discoveryService.searchByDSL(dslQuery);
+
+ JSONObject response = new DSLJSONResponseBuilder().results(jsonResultStr)
+ .query(dslQuery)
+ .build();
+
+ return Response.ok(response)
+ .build();
+ } catch (DiscoveryException | IllegalArgumentException e) {
+ LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch(Throwable e) {
+ LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Search using raw gremlin query format.
+ *
+ * @param gremlinQuery search query in raw gremlin format.
+ * @return JSON representing the type and results.
+ */
+ @GET
+ @Path("search/gremlin")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response searchUsingGremlinQuery(@QueryParam("query") String gremlinQuery) {
+ try {
+ ParamChecker.notEmpty(gremlinQuery, "gremlinQuery cannot be null or empty");
+ final List<Map<String, String>> results = discoveryService
+ .searchByGremlin(gremlinQuery);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.QUERY, gremlinQuery);
+ response.put(MetadataServiceClient.QUERY_TYPE, QUERY_TYPE_GREMLIN);
+
+ JSONArray list = new JSONArray();
+ for (Map<String, String> result : results) {
+ list.put(new JSONObject(result));
+ }
+ response.put(MetadataServiceClient.RESULTS, list);
+ response.put(MetadataServiceClient.COUNT, list.length());
+
+ return Response.ok(response)
+ .build();
+ } catch (DiscoveryException | IllegalArgumentException e) {
+ LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch(Throwable e) {
+ LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Search using full text search.
+ *
+ * @param query search query.
+ * @return JSON representing the type and results.
+ */
+ @GET
+ @Path("search/fulltext")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response searchUsingFullText(@QueryParam("query") String query) {
+ try {
+ ParamChecker.notEmpty(query, "query cannot be null or empty");
+ final String jsonResultStr = discoveryService.searchByFullText(query);
+ JSONArray rowsJsonArr = new JSONArray(jsonResultStr);
+
+ JSONObject response = new FullTextJSonResponseBuilder().results(rowsJsonArr)
+ .query(query)
+ .build();
+ return Response.ok(response)
+ .build();
+ } catch (DiscoveryException | IllegalArgumentException e) {
+ LOG.error("Unable to get entity list for query {}", query, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch(Throwable e) {
+ LOG.error("Unable to get entity list for query {}", query, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ private class JsonResponseBuilder {
+
+ protected int count = 0;
+ protected String query;
+ protected String queryType;
+ protected JSONObject response;
+
+ JsonResponseBuilder() {
+ this.response = new JSONObject();
+ }
+
+ protected JsonResponseBuilder count(int count) {
+ this.count = count;
+ return this;
+ }
+
+ public JsonResponseBuilder query(String query) {
+ this.query = query;
+ return this;
+ }
+
+ public JsonResponseBuilder queryType(String queryType) {
+ this.queryType = queryType;
+ return this;
+ }
+
+ protected JSONObject build() throws JSONException {
+
+ Preconditions.checkNotNull(query, "Query cannot be null");
+ Preconditions.checkNotNull(queryType, "Query Type must be specified");
+ Preconditions.checkArgument(count >= 0, "Search Result count should be > 0");
+
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.QUERY, query);
+ response.put(MetadataServiceClient.QUERY_TYPE, queryType);
+ response.put(MetadataServiceClient.COUNT, count);
+ return response;
+ }
+ }
+
+ private class DSLJSONResponseBuilder extends JsonResponseBuilder {
+
+ DSLJSONResponseBuilder() {
+ super();
+ }
+
+ private JSONObject dslResults;
+
+ public DSLJSONResponseBuilder results(JSONObject dslResults) {
+ this.dslResults = dslResults;
+ return this;
+ }
+
+ public DSLJSONResponseBuilder results(String dslResults) throws JSONException {
+ return results(new JSONObject(dslResults));
+ }
+
+ @Override
+ public JSONObject build() throws JSONException {
+ Preconditions.checkNotNull(dslResults);
+ JSONArray rowsJsonArr = dslResults.getJSONArray(MetadataServiceClient.ROWS);
+ count(rowsJsonArr.length());
+ queryType(QUERY_TYPE_DSL);
+ JSONObject response = super.build();
+ response.put(MetadataServiceClient.RESULTS, dslResults);
+ return response;
+ }
+
+ }
+
+ private class FullTextJSonResponseBuilder extends JsonResponseBuilder {
+
+ private JSONArray fullTextResults;
+
+ public FullTextJSonResponseBuilder results(JSONArray fullTextResults) {
+ this.fullTextResults = fullTextResults;
+ return this;
+ }
+
+ public FullTextJSonResponseBuilder results(String dslResults) throws JSONException {
+ return results(new JSONArray(dslResults));
+ }
+
+ public FullTextJSonResponseBuilder() {
+ super();
+ }
+
+ @Override
+ public JSONObject build() throws JSONException {
+ Preconditions.checkNotNull(fullTextResults);
+ count(fullTextResults.length());
+ queryType(QUERY_TYPE_FULLTEXT);
+
+ JSONObject response = super.build();
+ response.put(MetadataServiceClient.RESULTS, fullTextResults);
+ return response;
+ }
+ }
+}
[58/58] [abbrv] incubator-atlas git commit: Merge branch 'dal' of
github.com:MPR-Global/metadata into apache-local
Posted by ve...@apache.org.
Merge branch 'dal' of github.com:MPR-Global/metadata into apache-local
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/7f2b49ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/7f2b49ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/7f2b49ba
Branch: refs/heads/master
Commit: 7f2b49ba82347a7efd2a782b65d132831458017d
Parents: 9071ce2 e426db2
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Tue Jun 16 16:03:08 2015 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Tue Jun 16 16:03:08 2015 -0700
----------------------------------------------------------------------
.gitignore | 4 +
.../org/apache/atlas/hive/hook/HiveHook.java | 17 ---
dashboard/v2/bower.json | 1 -
dashboard/v2/public/index.html | 5 +-
.../public/modules/details/detailsController.js | 8 +-
.../public/modules/details/views/details.html | 6 +-
.../public/modules/lineage/lineageController.js | 113 +++++++++++++++++--
.../atlas/services/DefaultMetadataService.java | 9 +-
.../resources/MetadataDiscoveryResource.java | 2 +-
.../MetadataDiscoveryJerseyResourceIT.java | 2 +-
10 files changed, 122 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
[37/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLHiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLHiveHookIT.java
deleted file mode 100755
index d295d16..0000000
--- a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLHiveHookIT.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.hook;
-
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
-import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
-import org.apache.hadoop.metadata.security.SecurityProperties;
-import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.webapp.WebAppContext;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.Files;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-public class SSLHiveHookIT {
- private static final String DGI_URL = "https://localhost:21443/";
- private Driver driver;
- private MetadataServiceClient dgiCLient;
- private SessionState ss;
- private Path jksPath;
- private String providerUrl;
- private TestSecureEmbeddedServer secureEmbeddedServer;
-
- class TestSecureEmbeddedServer extends SecureEmbeddedServer {
-
- public TestSecureEmbeddedServer(int port, String path) throws IOException {
- super(port, path);
- }
-
- public Server getServer () { return server; }
-
- @Override
- public PropertiesConfiguration getConfiguration() {
- return super.getConfiguration();
- }
- }
-
- @BeforeClass
- public void setUp() throws Exception {
- //Set-up hive session
- HiveConf conf = getHiveConf();
- driver = new Driver(conf);
- ss = new SessionState(conf, System.getProperty("user.name"));
- ss = SessionState.start(ss);
- SessionState.setCurrentSessionState(ss);
-
- jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
-
- String persistDir = null;
- URL resource = SSLHiveHookIT.class.getResource("/");
- if (resource != null) {
- persistDir = resource.toURI().getPath();
- }
- // delete prior ssl-client.xml file
- resource = SSLHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
- if (resource != null) {
- File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
- if (sslClientFile != null && sslClientFile.exists()) {
- sslClientFile.delete();
- }
- }
- setupCredentials();
-
- final PropertiesConfiguration configuration = new PropertiesConfiguration();
- configuration.setProperty(TLS_ENABLED, true);
- configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/metadata.keystore");
- configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/metadata.keystore");
- configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
- configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
-
- configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
-
- dgiCLient = new MetadataServiceClient(DGI_URL) {
- @Override
- protected PropertiesConfiguration getClientProperties() throws MetadataException {
- return configuration;
- }
- };
-
- secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
- @Override
- public PropertiesConfiguration getConfiguration() {
- return configuration;
- }
- };
- WebAppContext webapp = new WebAppContext();
- webapp.setContextPath("/");
- webapp.setWar(System.getProperty("user.dir") + getWarPath());
- secureEmbeddedServer.getServer().setHandler(webapp);
-
- secureEmbeddedServer.getServer().start();
-
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- if (secureEmbeddedServer != null) {
- secureEmbeddedServer.getServer().stop();
- }
- }
-
- protected void setupCredentials() throws Exception {
- Configuration conf = new Configuration(false);
-
- File file = new File(jksPath.toUri().getPath());
- file.delete();
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- // create new aliases
- try {
-
- char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- KEYSTORE_PASSWORD_KEY, storepass);
-
- char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- TRUSTSTORE_PASSWORD_KEY, trustpass);
-
- char[] trustpass2 = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- "ssl.client.truststore.password", trustpass2);
-
- char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- SERVER_CERT_PASSWORD_KEY, certpass);
-
- // write out so that it can be found in checks
- provider.flush();
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- protected String getWarPath() {
- return String.format("/../../webapp/target/metadata-webapp-%s",
- System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
- }
-
- private HiveConf getHiveConf() {
- HiveConf hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
- hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore");
- hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
- hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
- hiveConf.set("hive.hook.dgi.synchronous", "true");
- return hiveConf;
- }
-
- private void runCommand(String cmd) throws Exception {
- ss.setCommandType(null);
- driver.run(cmd);
- }
-
- @Test
- public void testCreateDatabase() throws Exception {
- String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
- runCommand("create database " + dbName);
-
- assertDatabaseIsRegistered(dbName);
- }
-
- private void assertDatabaseIsRegistered(String dbName) throws Exception {
- assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
- }
-
- private void assertInstanceIsRegistered(String typeName, String colName, String colValue) throws Exception{
- JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
- Assert.assertEquals(results.length(), 1);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/atlas/MetadataServiceClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/MetadataServiceClient.java b/client/src/main/java/org/apache/atlas/MetadataServiceClient.java
new file mode 100755
index 0000000..675e28a
--- /dev/null
+++ b/client/src/main/java/org/apache/atlas/MetadataServiceClient.java
@@ -0,0 +1,368 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.atlas.security.SecureClientUtils;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
+
+/**
+ * Client for metadata.
+ */
+public class MetadataServiceClient {
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceClient.class);
+ public static final String NAME = "name";
+ public static final String GUID = "GUID";
+ public static final String TYPENAME = "typeName";
+
+ public static final String DEFINITION = "definition";
+ public static final String ERROR = "error";
+ public static final String STACKTRACE = "stackTrace";
+ public static final String REQUEST_ID = "requestId";
+ public static final String RESULTS = "results";
+ public static final String COUNT = "count";
+ public static final String ROWS = "rows";
+
+ public static final String BASE_URI = "api/atlas/";
+ public static final String TYPES = "types";
+ public static final String URI_ENTITIES = "entities";
+ public static final String URI_TRAITS = "traits";
+ public static final String URI_SEARCH = "discovery/search";
+ public static final String URI_LINEAGE = "lineage/hive/table";
+
+ public static final String QUERY = "query";
+ public static final String QUERY_TYPE = "queryType";
+ public static final String ATTRIBUTE_NAME = "property";
+ public static final String ATTRIBUTE_VALUE = "value";
+
+
+ public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure";
+ public static final String DATA_SET_SUPER_TYPE = "DataSet";
+ public static final String PROCESS_SUPER_TYPE = "Process";
+
+ public static final String JSON_MEDIA_TYPE = MediaType.APPLICATION_JSON + "; charset=UTF-8";
+
+ private WebResource service;
+
+ public MetadataServiceClient(String baseUrl) {
+ DefaultClientConfig config = new DefaultClientConfig();
+ PropertiesConfiguration clientConfig = null;
+ try {
+ clientConfig = getClientProperties();
+ if (clientConfig.getBoolean(TLS_ENABLED, false)) {
+ // create an SSL properties configuration if one doesn't exist. SSLFactory expects a file, so forced to create a
+ // configuration object, persist it, then subsequently pass in an empty configuration to SSLFactory
+ SecureClientUtils.persistSSLClientConfiguration(clientConfig);
+ }
+ } catch (Exception e) {
+ LOG.info("Error processing client configuration.", e);
+ }
+
+ URLConnectionClientHandler handler = SecureClientUtils.getClientConnectionHandler(config, clientConfig);
+
+ Client client = new Client(handler, config);
+ client.resource(UriBuilder.fromUri(baseUrl).build());
+
+ service = client.resource(UriBuilder.fromUri(baseUrl).build());
+ }
+
+ protected PropertiesConfiguration getClientProperties() throws MetadataException {
+ return PropertiesUtil.getClientProperties();
+ }
+
+ enum API {
+
+ //Type operations
+ CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST),
+ GET_TYPE(BASE_URI + TYPES, HttpMethod.GET),
+ LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET),
+ LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET),
+
+ //Entity operations
+ CREATE_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.POST),
+ GET_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.GET),
+ UPDATE_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.PUT),
+ LIST_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.GET),
+
+ //Trait operations
+ ADD_TRAITS(BASE_URI + URI_TRAITS, HttpMethod.POST),
+ DELETE_TRAITS(BASE_URI + URI_TRAITS, HttpMethod.DELETE),
+ LIST_TRAITS(BASE_URI + URI_TRAITS, HttpMethod.GET),
+
+ //Search operations
+ SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET),
+ SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET),
+ SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET),
+ SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET),
+
+ //Lineage operations
+ LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
+ LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
+ LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET);
+
+ private final String method;
+ private final String path;
+
+ API(String path, String method) {
+ this.path = path;
+ this.method = method;
+ }
+
+ public String getMethod() {
+ return method;
+ }
+
+ public String getPath() {
+ return path;
+ }
+ }
+
+ /**
+ * Register the given type(meta model)
+ * @param typeAsJson type definition a jaon
+ * @return result json object
+ * @throws MetadataServiceException
+ */
+ public JSONObject createType(String typeAsJson) throws MetadataServiceException {
+ return callAPI(API.CREATE_TYPE, typeAsJson);
+ }
+
+ public List<String> listTypes() throws MetadataServiceException {
+ try {
+ final JSONObject jsonObject = callAPI(API.LIST_TYPES, null);
+ final JSONArray list = jsonObject.getJSONArray(MetadataServiceClient.RESULTS);
+ ArrayList<String> types = new ArrayList<>();
+ for (int index = 0; index < list.length(); index++) {
+ types.add(list.getString(index));
+ }
+
+ return types;
+ } catch (JSONException e) {
+ throw new MetadataServiceException(API.LIST_TYPES, e);
+ }
+ }
+
+ public String getType(String typeName) throws MetadataServiceException {
+ WebResource resource = getResource(API.GET_TYPE, typeName);
+ try {
+ JSONObject response = callAPIWithResource(API.GET_TYPE, resource);
+ return response.getString(DEFINITION);
+ } catch (MetadataServiceException e) {
+ if (e.getStatus() == ClientResponse.Status.NOT_FOUND) {
+ return null;
+ }
+ throw e;
+ } catch (JSONException e) {
+ throw new MetadataServiceException(e);
+ }
+ }
+
+ /**
+ * Create the given entity
+ * @param entityAsJson entity(type instance) as json
+ * @return result json object
+ * @throws MetadataServiceException
+ */
+ public JSONObject createEntity(String entityAsJson) throws MetadataServiceException {
+ return callAPI(API.CREATE_ENTITY, entityAsJson);
+ }
+
+ /**
+ * Get an entity given the entity id
+ * @param guid entity id
+ * @return result json object
+ * @throws MetadataServiceException
+ */
+ public Referenceable getEntity(String guid) throws MetadataServiceException {
+ JSONObject jsonResponse = callAPI(API.GET_ENTITY, null, guid);
+ try {
+ String entityInstanceDefinition = jsonResponse.getString(MetadataServiceClient.DEFINITION);
+ return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true);
+ } catch (JSONException e) {
+ throw new MetadataServiceException(e);
+ }
+ }
+
+ /**
+ * Updates property for the entity corresponding to guid
+ * @param guid guid
+ * @param property property key
+ * @param value property value
+ */
+ public JSONObject updateEntity(String guid, String property, String value) throws MetadataServiceException {
+ WebResource resource = getResource(API.UPDATE_ENTITY, guid);
+ resource = resource.queryParam(ATTRIBUTE_NAME, property);
+ resource = resource.queryParam(ATTRIBUTE_VALUE, value);
+ return callAPIWithResource(API.UPDATE_ENTITY, resource);
+ }
+
+ public JSONObject searchEntity(String searchQuery) throws MetadataServiceException {
+ WebResource resource = getResource(API.SEARCH);
+ resource = resource.queryParam(QUERY, searchQuery);
+ return callAPIWithResource(API.SEARCH, resource);
+ }
+
+ /**
+ * Search given type name, an attribute and its value. Uses search dsl
+ * @param typeName name of the entity type
+ * @param attributeName attribute name
+ * @param attributeValue attribute value
+ * @return result json object
+ * @throws MetadataServiceException
+ */
+ public JSONArray rawSearch(String typeName, String attributeName, Object attributeValue) throws
+ MetadataServiceException {
+// String gremlinQuery = String.format(
+// "g.V.has(\"typeName\",\"%s\").and(_().has(\"%s.%s\", T.eq, \"%s\")).toList()",
+// typeName, typeName, attributeName, attributeValue);
+// return searchByGremlin(gremlinQuery);
+ String dslQuery = String.format("%s where %s = \"%s\"", typeName, attributeName, attributeValue);
+ return searchByDSL(dslQuery);
+ }
+
+ /**
+ * Search given query DSL
+ * @param query DSL query
+ * @return result json object
+ * @throws MetadataServiceException
+ */
+ public JSONArray searchByDSL(String query) throws MetadataServiceException {
+ LOG.debug("DSL query: {}", query);
+ WebResource resource = getResource(API.SEARCH_DSL);
+ resource = resource.queryParam(QUERY, query);
+ JSONObject result = callAPIWithResource(API.SEARCH_DSL, resource);
+ try {
+ return result.getJSONObject(RESULTS).getJSONArray(ROWS);
+ } catch (JSONException e) {
+ throw new MetadataServiceException(e);
+ }
+ }
+
+ /**
+ * Search given gremlin query
+ * @param gremlinQuery Gremlin query
+ * @return result json object
+ * @throws MetadataServiceException
+ */
+ public JSONObject searchByGremlin(String gremlinQuery) throws MetadataServiceException {
+ LOG.debug("Gremlin query: " + gremlinQuery);
+ WebResource resource = getResource(API.SEARCH_GREMLIN);
+ resource = resource.queryParam(QUERY, gremlinQuery);
+ return callAPIWithResource(API.SEARCH_GREMLIN, resource);
+ }
+
+ /**
+ * Search given full text search
+ * @param query Query
+ * @return result json object
+ * @throws MetadataServiceException
+ */
+ public JSONObject searchByFullText(String query) throws MetadataServiceException {
+ WebResource resource = getResource(API.SEARCH_FULL_TEXT);
+ resource = resource.queryParam(QUERY, query);
+ return callAPIWithResource(API.SEARCH_FULL_TEXT, resource);
+ }
+
+ public JSONObject getInputGraph(String datasetName) throws MetadataServiceException {
+ JSONObject response = callAPI(API.LINEAGE_INPUTS_GRAPH, null, datasetName, "/inputs/graph");
+ try {
+ return response.getJSONObject(MetadataServiceClient.RESULTS);
+ } catch (JSONException e) {
+ throw new MetadataServiceException(e);
+ }
+ }
+
+ public JSONObject getOutputGraph(String datasetName) throws MetadataServiceException {
+ JSONObject response = callAPI(API.LINEAGE_OUTPUTS_GRAPH, null, datasetName, "/outputs/graph");
+ try {
+ return response.getJSONObject(MetadataServiceClient.RESULTS);
+ } catch (JSONException e) {
+ throw new MetadataServiceException(e);
+ }
+ }
+
+ public String getRequestId(JSONObject json) throws MetadataServiceException {
+ try {
+ return json.getString(REQUEST_ID);
+ } catch (JSONException e) {
+ throw new MetadataServiceException(e);
+ }
+ }
+
+ private WebResource getResource(API api, String... pathParams) {
+ WebResource resource = service.path(api.getPath());
+ if (pathParams != null) {
+ for (String pathParam : pathParams) {
+ resource = resource.path(pathParam);
+ }
+ }
+ return resource;
+ }
+
+ private JSONObject callAPIWithResource(API api, WebResource resource) throws MetadataServiceException {
+ return callAPIWithResource(api, resource, null);
+ }
+
+ private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject)
+ throws MetadataServiceException {
+ ClientResponse clientResponse = resource
+ .accept(JSON_MEDIA_TYPE)
+ .type(JSON_MEDIA_TYPE)
+ .method(api.getMethod(), ClientResponse.class, requestObject);
+
+ Response.Status expectedStatus = HttpMethod.POST.equals(api.getMethod())
+ ? Response.Status.CREATED : Response.Status.OK;
+ if (clientResponse.getStatus() == expectedStatus.getStatusCode()) {
+ String responseAsString = clientResponse.getEntity(String.class);
+ try {
+ return new JSONObject(responseAsString);
+ } catch (JSONException e) {
+ throw new MetadataServiceException(api, e);
+ }
+ }
+
+ throw new MetadataServiceException(api, clientResponse);
+ }
+
+ private JSONObject callAPI(API api, Object requestObject,
+ String... pathParams) throws MetadataServiceException {
+ WebResource resource = getResource(api, pathParams);
+ return callAPIWithResource(api, resource, requestObject);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/atlas/MetadataServiceException.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/MetadataServiceException.java b/client/src/main/java/org/apache/atlas/MetadataServiceException.java
new file mode 100755
index 0000000..89ba89f
--- /dev/null
+++ b/client/src/main/java/org/apache/atlas/MetadataServiceException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import com.sun.jersey.api.client.ClientResponse;
+
+public class MetadataServiceException extends Exception {
+ private ClientResponse.Status status;
+
+ public MetadataServiceException(MetadataServiceClient.API api, Exception e) {
+ super("Metadata service API " + api + " failed", e);
+ }
+
+ public MetadataServiceException(MetadataServiceClient.API api, ClientResponse response) {
+ super("Metadata service API " + api + " failed with status " +
+ response.getClientResponseStatus().getStatusCode() + "(" +
+ response.getClientResponseStatus().getReasonPhrase() + ") Response Body (" +
+ response.getEntity(String.class) + ")");
+ this.status = response.getClientResponseStatus();
+ }
+
+ public MetadataServiceException(Exception e) {
+ super(e);
+ }
+
+ public ClientResponse.Status getStatus() {
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/atlas/PropertiesUtil.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/PropertiesUtil.java b/client/src/main/java/org/apache/atlas/PropertiesUtil.java
new file mode 100644
index 0000000..c1e014a
--- /dev/null
+++ b/client/src/main/java/org/apache/atlas/PropertiesUtil.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URL;
+
+public class PropertiesUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(PropertiesUtil.class);
+
+ private static final String APPLICATION_PROPERTIES = "application.properties";
+ public static final String CLIENT_PROPERTIES = "client.properties";
+
+ public static PropertiesConfiguration getApplicationProperties() throws MetadataException {
+ return getPropertiesConfiguration(APPLICATION_PROPERTIES);
+ }
+
+ public static PropertiesConfiguration getClientProperties() throws MetadataException {
+ return getPropertiesConfiguration(CLIENT_PROPERTIES);
+ }
+
+ private static PropertiesConfiguration getPropertiesConfiguration(String name) throws MetadataException {
+ String confLocation = System.getProperty("atlas.conf");
+ URL url;
+ try {
+ if (confLocation == null) {
+ url = PropertiesUtil.class.getResource("/" + name);
+ } else {
+ url = new File(confLocation, name).toURI().toURL();
+ }
+ LOG.info("Loading {} from {}", name, url);
+ return new PropertiesConfiguration(url);
+ } catch (Exception e) {
+ throw new MetadataException("Failed to load application properties", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java b/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java
new file mode 100644
index 0000000..608b003
--- /dev/null
+++ b/client/src/main/java/org/apache/atlas/security/SecureClientUtils.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.security;
+
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.PropertiesUtil;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
+import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
+import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
+
+import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
+import static org.apache.atlas.security.SecurityProperties.CLIENT_AUTH_KEY;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
+
+/**
+ *
+ */
+public class SecureClientUtils {
+
+ public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
+ private static final Logger LOG = LoggerFactory.getLogger(SecureClientUtils.class);
+
+
+ public static URLConnectionClientHandler getClientConnectionHandler(DefaultClientConfig config,
+ PropertiesConfiguration clientConfig) {
+ config.getProperties().put(
+ URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND,
+ true);
+ Configuration conf = new Configuration(false);
+ conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, "ssl-client.xml"));
+ String authType = "simple";
+ if (clientConfig != null) {
+ authType = clientConfig.getString("atlas.http.authentication.type", "simple");
+ }
+ UserGroupInformation.setConfiguration(conf);
+ final ConnectionConfigurator connConfigurator = newConnConfigurator(conf);
+ Authenticator authenticator = new PseudoDelegationTokenAuthenticator();
+ if (!authType.equals("simple")) {
+ authenticator = new KerberosDelegationTokenAuthenticator();
+ }
+ authenticator.setConnectionConfigurator(connConfigurator);
+ final DelegationTokenAuthenticator finalAuthenticator = (DelegationTokenAuthenticator) authenticator;
+ final DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token();
+ HttpURLConnectionFactory httpURLConnectionFactory = new HttpURLConnectionFactory() {
+ @Override
+ public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
+ try {
+ return new DelegationTokenAuthenticatedURL(finalAuthenticator, connConfigurator)
+ .openConnection(url, token, null);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ };
+
+ return new URLConnectionClientHandler(httpURLConnectionFactory);
+ }
+
+ private final static ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR =
+ new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT);
+ return conn;
+ }
+ };
+
+ private static ConnectionConfigurator newConnConfigurator(Configuration conf) {
+ try {
+ return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+ } catch (Exception e) {
+ LOG.debug("Cannot load customized ssl related configuration. " +
+ "Fallback to system-generic settings.", e);
+ return DEFAULT_TIMEOUT_CONN_CONFIGURATOR;
+ }
+ }
+
+ private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+ Configuration conf) throws IOException, GeneralSecurityException {
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
+
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+
+ return new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection c = (HttpsURLConnection) conn;
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ }
+ setTimeouts(conn, timeout);
+ return conn;
+ }
+ };
+ }
+
+ private static void setTimeouts(URLConnection connection, int socketTimeout) {
+ connection.setConnectTimeout(socketTimeout);
+ connection.setReadTimeout(socketTimeout);
+ }
+
+ private static File getSSLClientFile() throws MetadataException {
+ String confLocation = System.getProperty("atlas.conf");
+ File sslDir;
+ try {
+ if (confLocation == null) {
+ String persistDir = null;
+ URL resource = PropertiesUtil.class.getResource("/");
+ if (resource != null) {
+ persistDir = resource.toURI().getPath();
+ }
+ assert persistDir != null;
+ sslDir = new File(persistDir);
+ } else {
+ sslDir = new File(confLocation);
+ }
+ LOG.info("ssl-client.xml will be created in {}", sslDir);
+ } catch (Exception e) {
+ throw new MetadataException("Failed to find client configuration directory", e);
+ }
+ return new File(sslDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
+ }
+
+ public static void persistSSLClientConfiguration(PropertiesConfiguration clientConfig) throws MetadataException, IOException {
+ //trust settings
+ Configuration configuration = new Configuration(false);
+ File sslClientFile = getSSLClientFile();
+ if (!sslClientFile.exists()) {
+ configuration.set("ssl.client.truststore.type", "jks");
+ configuration.set("ssl.client.truststore.location", clientConfig.getString(TRUSTSTORE_FILE_KEY));
+ if (clientConfig.getBoolean(CLIENT_AUTH_KEY, false)) {
+ // need to get client key properties
+ configuration.set("ssl.client.keystore.location", clientConfig.getString(KEYSTORE_FILE_KEY));
+ configuration.set("ssl.client.keystore.type", "jks");
+ }
+ // add the configured credential provider
+ configuration.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ clientConfig.getString(CERT_STORES_CREDENTIAL_PROVIDER_PATH));
+ String hostnameVerifier = clientConfig.getString(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY);
+ if (hostnameVerifier != null) {
+ configuration.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, hostnameVerifier);
+ }
+
+ configuration.writeXml(new FileWriter(sslClientFile));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/atlas/security/SecurityProperties.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/security/SecurityProperties.java b/client/src/main/java/org/apache/atlas/security/SecurityProperties.java
new file mode 100644
index 0000000..e03ffcc
--- /dev/null
+++ b/client/src/main/java/org/apache/atlas/security/SecurityProperties.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.security;
+
+/**
+ *
+ */
+public interface SecurityProperties {
+ public static final String TLS_ENABLED = "atlas.enableTLS";
+ public static final String KEYSTORE_FILE_KEY = "keystore.file";
+ public static final String DEFAULT_KEYSTORE_FILE_LOCATION = "target/atlas.keystore";
+ public static final String KEYSTORE_PASSWORD_KEY = "keystore.password";
+ public static final String TRUSTSTORE_FILE_KEY = "truststore.file";
+ public static final String DEFATULT_TRUSTORE_FILE_LOCATION = "target/atlas.keystore";
+ public static final String TRUSTSTORE_PASSWORD_KEY = "truststore.password";
+ public static final String SERVER_CERT_PASSWORD_KEY = "password";
+ public static final String CLIENT_AUTH_KEY = "client.auth.enabled";
+ public static final String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path";
+ public static final String SSL_CLIENT_PROPERTIES = "ssl-client.xml";
+ public static final String BIND_ADDRESS = "atlas.server.bind.address";
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java b/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
deleted file mode 100755
index 6878ebe..0000000
--- a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceClient.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.metadata.security.SecureClientUtils;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriBuilder;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.TLS_ENABLED;
-
-/**
- * Client for metadata.
- */
-public class MetadataServiceClient {
- private static final Logger LOG = LoggerFactory.getLogger(MetadataServiceClient.class);
- public static final String NAME = "name";
- public static final String GUID = "GUID";
- public static final String TYPENAME = "typeName";
-
- public static final String DEFINITION = "definition";
- public static final String ERROR = "error";
- public static final String STACKTRACE = "stackTrace";
- public static final String REQUEST_ID = "requestId";
- public static final String RESULTS = "results";
- public static final String COUNT = "count";
- public static final String ROWS = "rows";
-
- public static final String BASE_URI = "api/metadata/";
- public static final String TYPES = "types";
- public static final String URI_ENTITIES = "entities";
- public static final String URI_TRAITS = "traits";
- public static final String URI_SEARCH = "discovery/search";
- public static final String URI_LINEAGE = "lineage/hive/table";
-
- public static final String QUERY = "query";
- public static final String QUERY_TYPE = "queryType";
- public static final String ATTRIBUTE_NAME = "property";
- public static final String ATTRIBUTE_VALUE = "value";
-
-
- public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure";
- public static final String DATA_SET_SUPER_TYPE = "DataSet";
- public static final String PROCESS_SUPER_TYPE = "Process";
-
- public static final String JSON_MEDIA_TYPE = MediaType.APPLICATION_JSON + "; charset=UTF-8";
-
- private WebResource service;
-
- public MetadataServiceClient(String baseUrl) {
- DefaultClientConfig config = new DefaultClientConfig();
- PropertiesConfiguration clientConfig = null;
- try {
- clientConfig = getClientProperties();
- if (clientConfig.getBoolean(TLS_ENABLED, false)) {
- // create an SSL properties configuration if one doesn't exist. SSLFactory expects a file, so forced to create a
- // configuration object, persist it, then subsequently pass in an empty configuration to SSLFactory
- SecureClientUtils.persistSSLClientConfiguration(clientConfig);
- }
- } catch (Exception e) {
- LOG.info("Error processing client configuration.", e);
- }
-
- URLConnectionClientHandler handler = SecureClientUtils.getClientConnectionHandler(config, clientConfig);
-
- Client client = new Client(handler, config);
- client.resource(UriBuilder.fromUri(baseUrl).build());
-
- service = client.resource(UriBuilder.fromUri(baseUrl).build());
- }
-
- protected PropertiesConfiguration getClientProperties() throws MetadataException {
- return PropertiesUtil.getClientProperties();
- }
-
- enum API {
-
- //Type operations
- CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST),
- GET_TYPE(BASE_URI + TYPES, HttpMethod.GET),
- LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET),
- LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET),
-
- //Entity operations
- CREATE_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.POST),
- GET_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.GET),
- UPDATE_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.PUT),
- LIST_ENTITY(BASE_URI + URI_ENTITIES, HttpMethod.GET),
-
- //Trait operations
- ADD_TRAITS(BASE_URI + URI_TRAITS, HttpMethod.POST),
- DELETE_TRAITS(BASE_URI + URI_TRAITS, HttpMethod.DELETE),
- LIST_TRAITS(BASE_URI + URI_TRAITS, HttpMethod.GET),
-
- //Search operations
- SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET),
- SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET),
- SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET),
- SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET),
-
- //Lineage operations
- LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
- LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
- LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET);
-
- private final String method;
- private final String path;
-
- API(String path, String method) {
- this.path = path;
- this.method = method;
- }
-
- public String getMethod() {
- return method;
- }
-
- public String getPath() {
- return path;
- }
- }
-
- /**
- * Register the given type(meta model)
- * @param typeAsJson type definition a jaon
- * @return result json object
- * @throws MetadataServiceException
- */
- public JSONObject createType(String typeAsJson) throws MetadataServiceException {
- return callAPI(API.CREATE_TYPE, typeAsJson);
- }
-
- public List<String> listTypes() throws MetadataServiceException {
- try {
- final JSONObject jsonObject = callAPI(API.LIST_TYPES, null);
- final JSONArray list = jsonObject.getJSONArray(MetadataServiceClient.RESULTS);
- ArrayList<String> types = new ArrayList<>();
- for (int index = 0; index < list.length(); index++) {
- types.add(list.getString(index));
- }
-
- return types;
- } catch (JSONException e) {
- throw new MetadataServiceException(API.LIST_TYPES, e);
- }
- }
-
- public String getType(String typeName) throws MetadataServiceException {
- WebResource resource = getResource(API.GET_TYPE, typeName);
- try {
- JSONObject response = callAPIWithResource(API.GET_TYPE, resource);
- return response.getString(DEFINITION);
- } catch (MetadataServiceException e) {
- if (e.getStatus() == ClientResponse.Status.NOT_FOUND) {
- return null;
- }
- throw e;
- } catch (JSONException e) {
- throw new MetadataServiceException(e);
- }
- }
-
- /**
- * Create the given entity
- * @param entityAsJson entity(type instance) as json
- * @return result json object
- * @throws MetadataServiceException
- */
- public JSONObject createEntity(String entityAsJson) throws MetadataServiceException {
- return callAPI(API.CREATE_ENTITY, entityAsJson);
- }
-
- /**
- * Get an entity given the entity id
- * @param guid entity id
- * @return result json object
- * @throws MetadataServiceException
- */
- public Referenceable getEntity(String guid) throws MetadataServiceException {
- JSONObject jsonResponse = callAPI(API.GET_ENTITY, null, guid);
- try {
- String entityInstanceDefinition = jsonResponse.getString(MetadataServiceClient.DEFINITION);
- return InstanceSerialization.fromJsonReferenceable(entityInstanceDefinition, true);
- } catch (JSONException e) {
- throw new MetadataServiceException(e);
- }
- }
-
- /**
- * Updates property for the entity corresponding to guid
- * @param guid guid
- * @param property property key
- * @param value property value
- */
- public JSONObject updateEntity(String guid, String property, String value) throws MetadataServiceException {
- WebResource resource = getResource(API.UPDATE_ENTITY, guid);
- resource = resource.queryParam(ATTRIBUTE_NAME, property);
- resource = resource.queryParam(ATTRIBUTE_VALUE, value);
- return callAPIWithResource(API.UPDATE_ENTITY, resource);
- }
-
- public JSONObject searchEntity(String searchQuery) throws MetadataServiceException {
- WebResource resource = getResource(API.SEARCH);
- resource = resource.queryParam(QUERY, searchQuery);
- return callAPIWithResource(API.SEARCH, resource);
- }
-
- /**
- * Search given type name, an attribute and its value. Uses search dsl
- * @param typeName name of the entity type
- * @param attributeName attribute name
- * @param attributeValue attribute value
- * @return result json object
- * @throws MetadataServiceException
- */
- public JSONArray rawSearch(String typeName, String attributeName, Object attributeValue) throws
- MetadataServiceException {
-// String gremlinQuery = String.format(
-// "g.V.has(\"typeName\",\"%s\").and(_().has(\"%s.%s\", T.eq, \"%s\")).toList()",
-// typeName, typeName, attributeName, attributeValue);
-// return searchByGremlin(gremlinQuery);
- String dslQuery = String.format("%s where %s = \"%s\"", typeName, attributeName, attributeValue);
- return searchByDSL(dslQuery);
- }
-
- /**
- * Search given query DSL
- * @param query DSL query
- * @return result json object
- * @throws MetadataServiceException
- */
- public JSONArray searchByDSL(String query) throws MetadataServiceException {
- LOG.debug("DSL query: {}", query);
- WebResource resource = getResource(API.SEARCH_DSL);
- resource = resource.queryParam(QUERY, query);
- JSONObject result = callAPIWithResource(API.SEARCH_DSL, resource);
- try {
- return result.getJSONObject(RESULTS).getJSONArray(ROWS);
- } catch (JSONException e) {
- throw new MetadataServiceException(e);
- }
- }
-
- /**
- * Search given gremlin query
- * @param gremlinQuery Gremlin query
- * @return result json object
- * @throws MetadataServiceException
- */
- public JSONObject searchByGremlin(String gremlinQuery) throws MetadataServiceException {
- LOG.debug("Gremlin query: " + gremlinQuery);
- WebResource resource = getResource(API.SEARCH_GREMLIN);
- resource = resource.queryParam(QUERY, gremlinQuery);
- return callAPIWithResource(API.SEARCH_GREMLIN, resource);
- }
-
- /**
- * Search given full text search
- * @param query Query
- * @return result json object
- * @throws MetadataServiceException
- */
- public JSONObject searchByFullText(String query) throws MetadataServiceException {
- WebResource resource = getResource(API.SEARCH_FULL_TEXT);
- resource = resource.queryParam(QUERY, query);
- return callAPIWithResource(API.SEARCH_FULL_TEXT, resource);
- }
-
- public JSONObject getInputGraph(String datasetName) throws MetadataServiceException {
- JSONObject response = callAPI(API.LINEAGE_INPUTS_GRAPH, null, datasetName, "/inputs/graph");
- try {
- return response.getJSONObject(MetadataServiceClient.RESULTS);
- } catch (JSONException e) {
- throw new MetadataServiceException(e);
- }
- }
-
- public JSONObject getOutputGraph(String datasetName) throws MetadataServiceException {
- JSONObject response = callAPI(API.LINEAGE_OUTPUTS_GRAPH, null, datasetName, "/outputs/graph");
- try {
- return response.getJSONObject(MetadataServiceClient.RESULTS);
- } catch (JSONException e) {
- throw new MetadataServiceException(e);
- }
- }
-
- public String getRequestId(JSONObject json) throws MetadataServiceException {
- try {
- return json.getString(REQUEST_ID);
- } catch (JSONException e) {
- throw new MetadataServiceException(e);
- }
- }
-
- private WebResource getResource(API api, String... pathParams) {
- WebResource resource = service.path(api.getPath());
- if (pathParams != null) {
- for (String pathParam : pathParams) {
- resource = resource.path(pathParam);
- }
- }
- return resource;
- }
-
- private JSONObject callAPIWithResource(API api, WebResource resource) throws MetadataServiceException {
- return callAPIWithResource(api, resource, null);
- }
-
- private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject)
- throws MetadataServiceException {
- ClientResponse clientResponse = resource
- .accept(JSON_MEDIA_TYPE)
- .type(JSON_MEDIA_TYPE)
- .method(api.getMethod(), ClientResponse.class, requestObject);
-
- Response.Status expectedStatus = HttpMethod.POST.equals(api.getMethod())
- ? Response.Status.CREATED : Response.Status.OK;
- if (clientResponse.getStatus() == expectedStatus.getStatusCode()) {
- String responseAsString = clientResponse.getEntity(String.class);
- try {
- return new JSONObject(responseAsString);
- } catch (JSONException e) {
- throw new MetadataServiceException(api, e);
- }
- }
-
- throw new MetadataServiceException(api, clientResponse);
- }
-
- private JSONObject callAPI(API api, Object requestObject,
- String... pathParams) throws MetadataServiceException {
- WebResource resource = getResource(api, pathParams);
- return callAPIWithResource(api, resource, requestObject);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java b/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java
deleted file mode 100755
index e22e079..0000000
--- a/client/src/main/java/org/apache/hadoop/metadata/MetadataServiceException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import com.sun.jersey.api.client.ClientResponse;
-
-public class MetadataServiceException extends Exception {
- private ClientResponse.Status status;
-
- public MetadataServiceException(MetadataServiceClient.API api, Exception e) {
- super("Metadata service API " + api + " failed", e);
- }
-
- public MetadataServiceException(MetadataServiceClient.API api, ClientResponse response) {
- super("Metadata service API " + api + " failed with status " +
- response.getClientResponseStatus().getStatusCode() + "(" +
- response.getClientResponseStatus().getReasonPhrase() + ") Response Body (" +
- response.getEntity(String.class) + ")");
- this.status = response.getClientResponseStatus();
- }
-
- public MetadataServiceException(Exception e) {
- super(e);
- }
-
- public ClientResponse.Status getStatus() {
- return status;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/hadoop/metadata/PropertiesUtil.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/hadoop/metadata/PropertiesUtil.java b/client/src/main/java/org/apache/hadoop/metadata/PropertiesUtil.java
deleted file mode 100644
index 2d5e0b5..0000000
--- a/client/src/main/java/org/apache/hadoop/metadata/PropertiesUtil.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.net.URL;
-
-public class PropertiesUtil {
- private static final Logger LOG = LoggerFactory.getLogger(PropertiesUtil.class);
-
- private static final String APPLICATION_PROPERTIES = "application.properties";
- public static final String CLIENT_PROPERTIES = "client.properties";
-
- public static PropertiesConfiguration getApplicationProperties() throws MetadataException {
- return getPropertiesConfiguration(APPLICATION_PROPERTIES);
- }
-
- public static PropertiesConfiguration getClientProperties() throws MetadataException {
- return getPropertiesConfiguration(CLIENT_PROPERTIES);
- }
-
- private static PropertiesConfiguration getPropertiesConfiguration(String name) throws MetadataException {
- String confLocation = System.getProperty("metadata.conf");
- URL url;
- try {
- if (confLocation == null) {
- url = PropertiesUtil.class.getResource("/" + name);
- } else {
- url = new File(confLocation, name).toURI().toURL();
- }
- LOG.info("Loading {} from {}", name, url);
- return new PropertiesConfiguration(url);
- } catch (Exception e) {
- throw new MetadataException("Failed to load application properties", e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/hadoop/metadata/security/SecureClientUtils.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/hadoop/metadata/security/SecureClientUtils.java b/client/src/main/java/org/apache/hadoop/metadata/security/SecureClientUtils.java
deleted file mode 100644
index 7755b82..0000000
--- a/client/src/main/java/org/apache/hadoop/metadata/security/SecureClientUtils.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.security;
-
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.client.Authenticator;
-import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
-import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
-import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLSocketFactory;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.net.URLConnection;
-import java.security.GeneralSecurityException;
-import java.security.PrivilegedExceptionAction;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-/**
- *
- */
-public class SecureClientUtils {
-
- public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
- private static final Logger LOG = LoggerFactory.getLogger(SecureClientUtils.class);
-
-
- public static URLConnectionClientHandler getClientConnectionHandler(DefaultClientConfig config,
- PropertiesConfiguration clientConfig) {
- config.getProperties().put(
- URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND,
- true);
- Configuration conf = new Configuration(false);
- conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, "ssl-client.xml"));
- String authType = "simple";
- if (clientConfig != null) {
- authType = clientConfig.getString("metadata.http.authentication.type", "simple");
- }
- UserGroupInformation.setConfiguration(conf);
- final ConnectionConfigurator connConfigurator = newConnConfigurator(conf);
- Authenticator authenticator = new PseudoDelegationTokenAuthenticator();
- if (!authType.equals("simple")) {
- authenticator = new KerberosDelegationTokenAuthenticator();
- }
- authenticator.setConnectionConfigurator(connConfigurator);
- final DelegationTokenAuthenticator finalAuthenticator = (DelegationTokenAuthenticator) authenticator;
- final DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token();
- HttpURLConnectionFactory httpURLConnectionFactory = new HttpURLConnectionFactory() {
- @Override
- public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
- try {
- return new DelegationTokenAuthenticatedURL(finalAuthenticator, connConfigurator)
- .openConnection(url, token, null);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
- };
-
- return new URLConnectionClientHandler(httpURLConnectionFactory);
- }
-
- private final static ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR =
- new ConnectionConfigurator() {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT);
- return conn;
- }
- };
-
- private static ConnectionConfigurator newConnConfigurator(Configuration conf) {
- try {
- return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
- } catch (Exception e) {
- LOG.debug("Cannot load customized ssl related configuration. " +
- "Fallback to system-generic settings.", e);
- return DEFAULT_TIMEOUT_CONN_CONFIGURATOR;
- }
- }
-
- private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
- Configuration conf) throws IOException, GeneralSecurityException {
- final SSLFactory factory;
- final SSLSocketFactory sf;
- final HostnameVerifier hv;
-
- factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
- factory.init();
- sf = factory.createSSLSocketFactory();
- hv = factory.getHostnameVerifier();
-
- return new ConnectionConfigurator() {
- @Override
- public HttpURLConnection configure(HttpURLConnection conn)
- throws IOException {
- if (conn instanceof HttpsURLConnection) {
- HttpsURLConnection c = (HttpsURLConnection) conn;
- c.setSSLSocketFactory(sf);
- c.setHostnameVerifier(hv);
- }
- setTimeouts(conn, timeout);
- return conn;
- }
- };
- }
-
- private static void setTimeouts(URLConnection connection, int socketTimeout) {
- connection.setConnectTimeout(socketTimeout);
- connection.setReadTimeout(socketTimeout);
- }
-
- private static File getSSLClientFile() throws MetadataException {
- String confLocation = System.getProperty("metadata.conf");
- File sslDir;
- try {
- if (confLocation == null) {
- String persistDir = null;
- URL resource = PropertiesUtil.class.getResource("/");
- if (resource != null) {
- persistDir = resource.toURI().getPath();
- }
- assert persistDir != null;
- sslDir = new File(persistDir);
- } else {
- sslDir = new File(confLocation);
- }
- LOG.info("ssl-client.xml will be created in {}", sslDir);
- } catch (Exception e) {
- throw new MetadataException("Failed to find client configuration directory", e);
- }
- return new File(sslDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
- }
-
- public static void persistSSLClientConfiguration(PropertiesConfiguration clientConfig) throws MetadataException, IOException {
- //trust settings
- Configuration configuration = new Configuration(false);
- File sslClientFile = getSSLClientFile();
- if (!sslClientFile.exists()) {
- configuration.set("ssl.client.truststore.type", "jks");
- configuration.set("ssl.client.truststore.location", clientConfig.getString(TRUSTSTORE_FILE_KEY));
- if (clientConfig.getBoolean(CLIENT_AUTH_KEY, false)) {
- // need to get client key properties
- configuration.set("ssl.client.keystore.location", clientConfig.getString(KEYSTORE_FILE_KEY));
- configuration.set("ssl.client.keystore.type", "jks");
- }
- // add the configured credential provider
- configuration.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- clientConfig.getString(CERT_STORES_CREDENTIAL_PROVIDER_PATH));
- String hostnameVerifier = clientConfig.getString(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY);
- if (hostnameVerifier != null) {
- configuration.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, hostnameVerifier);
- }
-
- configuration.writeXml(new FileWriter(sslClientFile));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/main/java/org/apache/hadoop/metadata/security/SecurityProperties.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/hadoop/metadata/security/SecurityProperties.java b/client/src/main/java/org/apache/hadoop/metadata/security/SecurityProperties.java
deleted file mode 100644
index df7ae52..0000000
--- a/client/src/main/java/org/apache/hadoop/metadata/security/SecurityProperties.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.security;
-
-/**
- *
- */
-public interface SecurityProperties {
- public static final String TLS_ENABLED = "metadata.enableTLS";
- public static final String KEYSTORE_FILE_KEY = "keystore.file";
- public static final String DEFAULT_KEYSTORE_FILE_LOCATION = "target/metadata.keystore";
- public static final String KEYSTORE_PASSWORD_KEY = "keystore.password";
- public static final String TRUSTSTORE_FILE_KEY = "truststore.file";
- public static final String DEFATULT_TRUSTORE_FILE_LOCATION = "target/metadata.keystore";
- public static final String TRUSTSTORE_PASSWORD_KEY = "truststore.password";
- public static final String SERVER_CERT_PASSWORD_KEY = "password";
- public static final String CLIENT_AUTH_KEY = "client.auth.enabled";
- public static final String CERT_STORES_CREDENTIAL_PROVIDER_PATH = "cert.stores.credential.provider.path";
- public static final String SSL_CLIENT_PROPERTIES = "ssl-client.xml";
- public static final String BIND_ADDRESS = "metadata.server.bind.address";
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java b/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java
new file mode 100644
index 0000000..13cce01
--- /dev/null
+++ b/client/src/test/java/org/apache/atlas/security/BaseSecurityTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.security;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.zookeeper.Environment;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.testng.Assert;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.util.Locale;
+import java.util.Properties;
+
+/**
+ *
+ */
+public class BaseSecurityTest {
+ private static final String JAAS_ENTRY =
+ "%s { \n"
+ + " %s required\n"
+ // kerberos module
+ + " keyTab=\"%s\"\n"
+ + " debug=true\n"
+ + " principal=\"%s\"\n"
+ + " useKeyTab=true\n"
+ + " useTicketCache=false\n"
+ + " doNotPrompt=true\n"
+ + " storeKey=true;\n"
+ + "}; \n";
+ protected MiniKdc kdc;
+
+ protected String getWarPath() {
+ return String.format("/target/atlas-webapp-%s.war",
+ System.getProperty("release.version", "0.1-incubating-SNAPSHOT"));
+ }
+
+ protected void generateTestProperties(Properties props) throws ConfigurationException, IOException {
+ PropertiesConfiguration config = new PropertiesConfiguration(System.getProperty("user.dir") +
+ "/../src/conf/application.properties");
+ for (String propName : props.stringPropertyNames()) {
+ config.setProperty(propName, props.getProperty(propName));
+ }
+ File file = new File(System.getProperty("user.dir"), "application.properties");
+ file.deleteOnExit();
+ Writer fileWriter = new FileWriter(file);
+ config.save(fileWriter);
+ }
+
+ protected void startEmbeddedServer(Server server) throws Exception {
+ WebAppContext webapp = new WebAppContext();
+ webapp.setContextPath("/");
+ webapp.setWar(System.getProperty("user.dir") + getWarPath());
+ server.setHandler(webapp);
+
+ server.start();
+ }
+
+ protected File startKDC() throws Exception {
+ File target = Files.createTempDirectory("sectest").toFile();
+ File kdcWorkDir = new File(target, "kdc");
+ Properties kdcConf = MiniKdc.createConf();
+ kdcConf.setProperty(MiniKdc.DEBUG, "true");
+ kdc = new MiniKdc(kdcConf, kdcWorkDir);
+ kdc.start();
+
+ Assert.assertNotNull(kdc.getRealm());
+ return kdcWorkDir;
+ }
+
+ public String createJAASEntry(
+ String context,
+ String principal,
+ File keytab) {
+ String keytabpath = keytab.getAbsolutePath();
+ // fix up for windows; no-op on unix
+ keytabpath = keytabpath.replace('\\', '/');
+ return String.format(
+ Locale.ENGLISH,
+ JAAS_ENTRY,
+ context,
+ getKerberosAuthModuleForJVM(),
+ keytabpath,
+ principal);
+ }
+
+ protected String getKerberosAuthModuleForJVM() {
+ if (System.getProperty("java.vendor").contains("IBM")) {
+ return "com.ibm.security.auth.module.Krb5LoginModule";
+ } else {
+ return "com.sun.security.auth.module.Krb5LoginModule";
+ }
+ }
+
+ protected void bindJVMtoJAASFile(File jaasFile) {
+ String path = jaasFile.getAbsolutePath();
+ System.setProperty(Environment.JAAS_CONF_KEY, path);
+ }
+
+ protected File createKeytab(MiniKdc kdc, File kdcWorkDir, String principal, String filename) throws Exception {
+ File keytab = new File(kdcWorkDir, filename);
+ kdc.createPrincipal(keytab,
+ principal,
+ principal + "/localhost",
+ principal + "/127.0.0.1");
+ return keytab;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/client/src/test/java/org/apache/hadoop/metadata/security/BaseSecurityTest.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/hadoop/metadata/security/BaseSecurityTest.java b/client/src/test/java/org/apache/hadoop/metadata/security/BaseSecurityTest.java
deleted file mode 100644
index ac099e8..0000000
--- a/client/src/test/java/org/apache/hadoop/metadata/security/BaseSecurityTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.security;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.minikdc.MiniKdc;
-import org.apache.zookeeper.Environment;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.webapp.WebAppContext;
-import org.testng.Assert;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.Writer;
-import java.nio.file.Files;
-import java.util.Locale;
-import java.util.Properties;
-
-/**
- *
- */
-public class BaseSecurityTest {
- private static final String JAAS_ENTRY =
- "%s { \n"
- + " %s required\n"
- // kerberos module
- + " keyTab=\"%s\"\n"
- + " debug=true\n"
- + " principal=\"%s\"\n"
- + " useKeyTab=true\n"
- + " useTicketCache=false\n"
- + " doNotPrompt=true\n"
- + " storeKey=true;\n"
- + "}; \n";
- protected MiniKdc kdc;
-
- protected String getWarPath() {
- return String.format("/target/metadata-webapp-%s.war",
- System.getProperty("release.version", "0.1-incubating-SNAPSHOT"));
- }
-
- protected void generateTestProperties(Properties props) throws ConfigurationException, IOException {
- PropertiesConfiguration config = new PropertiesConfiguration(System.getProperty("user.dir") +
- "/../src/conf/application.properties");
- for (String propName : props.stringPropertyNames()) {
- config.setProperty(propName, props.getProperty(propName));
- }
- File file = new File(System.getProperty("user.dir"), "application.properties");
- file.deleteOnExit();
- Writer fileWriter = new FileWriter(file);
- config.save(fileWriter);
- }
-
- protected void startEmbeddedServer(Server server) throws Exception {
- WebAppContext webapp = new WebAppContext();
- webapp.setContextPath("/");
- webapp.setWar(System.getProperty("user.dir") + getWarPath());
- server.setHandler(webapp);
-
- server.start();
- }
-
- protected File startKDC() throws Exception {
- File target = Files.createTempDirectory("sectest").toFile();
- File kdcWorkDir = new File(target, "kdc");
- Properties kdcConf = MiniKdc.createConf();
- kdcConf.setProperty(MiniKdc.DEBUG, "true");
- kdc = new MiniKdc(kdcConf, kdcWorkDir);
- kdc.start();
-
- Assert.assertNotNull(kdc.getRealm());
- return kdcWorkDir;
- }
-
- public String createJAASEntry(
- String context,
- String principal,
- File keytab) {
- String keytabpath = keytab.getAbsolutePath();
- // fix up for windows; no-op on unix
- keytabpath = keytabpath.replace('\\', '/');
- return String.format(
- Locale.ENGLISH,
- JAAS_ENTRY,
- context,
- getKerberosAuthModuleForJVM(),
- keytabpath,
- principal);
- }
-
- protected String getKerberosAuthModuleForJVM() {
- if (System.getProperty("java.vendor").contains("IBM")) {
- return "com.ibm.security.auth.module.Krb5LoginModule";
- } else {
- return "com.sun.security.auth.module.Krb5LoginModule";
- }
- }
-
- protected void bindJVMtoJAASFile(File jaasFile) {
- String path = jaasFile.getAbsolutePath();
- System.setProperty(Environment.JAAS_CONF_KEY, path);
- }
-
- protected File createKeytab(MiniKdc kdc, File kdcWorkDir, String principal, String filename) throws Exception {
- File keytab = new File(kdcWorkDir, filename);
- kdc.createPrincipal(keytab,
- principal,
- principal + "/localhost",
- principal + "/127.0.0.1");
- return keytab;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/GraphTransaction.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/GraphTransaction.java b/repository/src/main/java/org/apache/atlas/GraphTransaction.java
new file mode 100644
index 0000000..2c3983f
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/GraphTransaction.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD)
+public @interface GraphTransaction {}
[06/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/RexsterGraphResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/RexsterGraphResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/RexsterGraphResource.java
new file mode 100755
index 0000000..7f8fd17
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/RexsterGraphResource.java
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Element;
+import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.blueprints.Vertex;
+import com.tinkerpop.blueprints.VertexQuery;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.web.util.Servlets;
+import org.apache.commons.lang.StringUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Jersey Resource for lineage metadata operations.
+ * Implements most of the GET operations of Rexster API with out the indexes.
+ * https://github.com/tinkerpop/rexster/wiki/Basic-REST-API
+ *
+ * This is a subset of Rexster's REST API, designed to provide only read-only methods
+ * for accessing the backend graph.
+ */
+@Path("graph")
+@Singleton
+public class RexsterGraphResource {
+ public static final String OUT_E = "outE";
+ public static final String IN_E = "inE";
+ public static final String BOTH_E = "bothE";
+ public static final String OUT = "out";
+ public static final String IN = "in";
+ public static final String BOTH = "both";
+ public static final String OUT_COUNT = "outCount";
+ public static final String IN_COUNT = "inCount";
+ public static final String BOTH_COUNT = "bothCount";
+ public static final String OUT_IDS = "outIds";
+ public static final String IN_IDS = "inIds";
+ public static final String BOTH_IDS = "bothIds";
+ private static final Logger LOG = LoggerFactory.getLogger(RexsterGraphResource.class);
+
+ private TitanGraph graph;
+
+ @Inject
+ public RexsterGraphResource(GraphProvider<TitanGraph> graphProvider) {
+ this.graph = graphProvider.get();
+ }
+
+ private static void validateInputs(String errorMsg, String... inputs) {
+ for (String input : inputs) {
+ if (StringUtils.isEmpty(input)) {
+ throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
+ .entity(errorMsg)
+ .type("text/plain")
+ .build());
+ }
+ }
+ }
+
+ protected Graph getGraph() {
+ return graph;
+ }
+
+ protected Set<String> getVertexIndexedKeys() {
+ return graph.getIndexedKeys(Vertex.class);
+ }
+
+ protected Set<String> getEdgeIndexedKeys() {
+ return graph.getIndexedKeys(Edge.class);
+ }
+
+ /**
+ * Get a single vertex with a unique id.
+ *
+ * GET http://host/metadata/lineage/vertices/id
+ * graph.getVertex(id);
+ */
+ @GET
+ @Path("/vertices/{id}")
+ @Produces({Servlets.JSON_MEDIA_TYPE})
+ public Response getVertex(@PathParam("id") final String vertexId) {
+ LOG.info("Get vertex for vertexId= {}", vertexId);
+ validateInputs("Invalid argument: vertex id passed is null or empty.", vertexId);
+ try {
+ Vertex vertex = findVertex(vertexId);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.RESULTS, GraphSONUtility.jsonFromElement(
+ vertex, getVertexIndexedKeys(), GraphSONMode.NORMAL));
+ return Response.ok(response).build();
+ } catch (JSONException e) {
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ private Vertex findVertex(String vertexId) {
+ Vertex vertex = getGraph().getVertex(vertexId);
+ if (vertex == null) {
+ String message = "Vertex with [" + vertexId + "] cannot be found.";
+ LOG.info(message);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(message, Response.Status.NOT_FOUND));
+ }
+
+ return vertex;
+ }
+
+ /**
+ * Get properties for a single vertex with a unique id.
+ * This is NOT a rexster API.
+ * <p/>
+ * GET http://host/metadata/lineage/vertices/properties/id
+ */
+ @GET
+ @Path("/vertices/properties/{id}")
+ @Produces({Servlets.JSON_MEDIA_TYPE})
+ public Response getVertexProperties(@PathParam("id") final String vertexId,
+ @DefaultValue("false") @QueryParam("relationships")
+ final String relationships) {
+ LOG.info("Get vertex for vertexId= {}", vertexId);
+ validateInputs("Invalid argument: vertex id passed is null or empty.", vertexId);
+ try {
+ Vertex vertex = findVertex(vertexId);
+
+ Map<String, String> vertexProperties = getVertexProperties(vertex);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.RESULTS, new JSONObject(vertexProperties));
+ response.put(MetadataServiceClient.COUNT, vertexProperties.size());
+ return Response.ok(response).build();
+ } catch (JSONException e) {
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ private Map<String, String> getVertexProperties(Vertex vertex) {
+ Map<String, String> vertexProperties = new HashMap<>();
+ for (String key : vertex.getPropertyKeys()) {
+ vertexProperties.put(key, vertex.<String>getProperty(key));
+ }
+
+ // todo: get the properties from relationships
+
+ return vertexProperties;
+ }
+
+ /**
+ * Get a list of vertices matching a property key and a value.
+ * <p/>
+ * GET http://host/metadata/lineage/vertices?key=<key>&value=<value>
+ * graph.getVertices(key, value);
+ */
+ @GET
+ @Path("/vertices")
+ @Produces({Servlets.JSON_MEDIA_TYPE})
+ public Response getVertices(@QueryParam("key") final String key,
+ @QueryParam("value") final String value) {
+ LOG.info("Get vertices for property key= {}, value= {}", key, value);
+ validateInputs("Invalid argument: key or value passed is null or empty.", key, value);
+ try {
+ JSONObject response = buildJSONResponse(getGraph().getVertices(key, value));
+ return Response.ok(response).build();
+
+ } catch (JSONException e) {
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Get a list of adjacent edges with a direction.
+ *
+ * GET http://host/metadata/lineage/vertices/id/direction
+ * graph.getVertex(id).get{Direction}Edges();
+ * direction: {(?!outE)(?!bothE)(?!inE)(?!out)(?!both)(?!in)(?!query).+}
+ */
+ @GET
+ @Path("vertices/{id}/{direction}")
+ @Produces({Servlets.JSON_MEDIA_TYPE})
+ public Response getVertexEdges(@PathParam("id") String vertexId,
+ @PathParam("direction") String direction) {
+ LOG.info("Get vertex edges for vertexId= {}, direction= {}", vertexId, direction);
+ // Validate vertex id. Direction is validated in VertexQueryArguments.
+ validateInputs("Invalid argument: vertex id or direction passed is null or empty.",
+ vertexId, direction);
+ try {
+ Vertex vertex = findVertex(vertexId);
+
+ return getVertexEdges(vertex, direction);
+
+ } catch (JSONException e) {
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ private Response getVertexEdges(Vertex vertex, String direction) throws JSONException {
+ // break out the segment into the return and the direction
+ VertexQueryArguments queryArguments = new VertexQueryArguments(direction);
+ // if this is a query and the _return is "count" then we don't bother to send back the
+ // result array
+ boolean countOnly = queryArguments.isCountOnly();
+ // what kind of data the calling client wants back (vertices, edges, count, vertex
+ // identifiers)
+ ReturnType returnType = queryArguments.getReturnType();
+ // the query direction (both, out, in)
+ Direction queryDirection = queryArguments.getQueryDirection();
+
+ VertexQuery query = vertex.query().direction(queryDirection);
+
+ JSONArray elementArray = new JSONArray();
+ long counter = 0;
+ if (returnType == ReturnType.VERTICES || returnType == ReturnType.VERTEX_IDS) {
+ Iterable<Vertex> vertexQueryResults = query.vertices();
+ for (Vertex v : vertexQueryResults) {
+ if (returnType.equals(ReturnType.VERTICES)) {
+ elementArray.put(GraphSONUtility.jsonFromElement(
+ v, getVertexIndexedKeys(), GraphSONMode.NORMAL));
+ } else {
+ elementArray.put(v.getId());
+ }
+ counter++;
+ }
+ } else if (returnType == ReturnType.EDGES) {
+ Iterable<Edge> edgeQueryResults = query.edges();
+ for (Edge e : edgeQueryResults) {
+ elementArray.put(GraphSONUtility.jsonFromElement(
+ e, getEdgeIndexedKeys(), GraphSONMode.NORMAL));
+ counter++;
+ }
+ } else if (returnType == ReturnType.COUNT) {
+ counter = query.count();
+ }
+
+ JSONObject response = new JSONObject();
+ if (!countOnly) {
+ response.put(MetadataServiceClient.RESULTS, elementArray);
+ }
+ response.put(MetadataServiceClient.COUNT, counter);
+ return Response.ok(response).build();
+ }
+
+ /**
+ * Get a single edge with a unique id.
+ *
+ * GET http://host/metadata/lineage/edges/id
+ * graph.getEdge(id);
+ */
+ @GET
+ @Path("/edges/{id}")
+ @Produces({Servlets.JSON_MEDIA_TYPE})
+ public Response getEdge(@PathParam("id") final String edgeId) {
+ LOG.info("Get vertex for edgeId= {}", edgeId);
+ validateInputs("Invalid argument: edge id passed is null or empty.", edgeId);
+ try {
+ Edge edge = getGraph().getEdge(edgeId);
+ if (edge == null) {
+ String message = "Edge with [" + edgeId + "] cannot be found.";
+ LOG.info(message);
+ throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND)
+ .entity(Servlets.escapeJsonString(message)).build());
+ }
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.RESULTS, GraphSONUtility.jsonFromElement(
+ edge, getEdgeIndexedKeys(), GraphSONMode.NORMAL));
+ return Response.ok(response).build();
+ } catch (JSONException e) {
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ private <T extends Element> JSONObject buildJSONResponse(Iterable<T> elements)
+ throws JSONException {
+ JSONArray vertexArray = new JSONArray();
+ long counter = 0;
+ for (Element element : elements) {
+ counter++;
+ vertexArray.put(GraphSONUtility.jsonFromElement(
+ element, getVertexIndexedKeys(), GraphSONMode.NORMAL));
+ }
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.RESULTS, vertexArray);
+ response.put(MetadataServiceClient.COUNT, counter);
+
+ return response;
+ }
+ private enum ReturnType {VERTICES, EDGES, COUNT, VERTEX_IDS}
+
+ /**
+ * Helper class for query arguments.
+ */
+ public static final class VertexQueryArguments {
+
+ private final Direction queryDirection;
+ private final ReturnType returnType;
+ private final boolean countOnly;
+
+ public VertexQueryArguments(String directionSegment) {
+ if (OUT_E.equals(directionSegment)) {
+ returnType = ReturnType.EDGES;
+ queryDirection = Direction.OUT;
+ countOnly = false;
+ } else if (IN_E.equals(directionSegment)) {
+ returnType = ReturnType.EDGES;
+ queryDirection = Direction.IN;
+ countOnly = false;
+ } else if (BOTH_E.equals(directionSegment)) {
+ returnType = ReturnType.EDGES;
+ queryDirection = Direction.BOTH;
+ countOnly = false;
+ } else if (OUT.equals(directionSegment)) {
+ returnType = ReturnType.VERTICES;
+ queryDirection = Direction.OUT;
+ countOnly = false;
+ } else if (IN.equals(directionSegment)) {
+ returnType = ReturnType.VERTICES;
+ queryDirection = Direction.IN;
+ countOnly = false;
+ } else if (BOTH.equals(directionSegment)) {
+ returnType = ReturnType.VERTICES;
+ queryDirection = Direction.BOTH;
+ countOnly = false;
+ } else if (BOTH_COUNT.equals(directionSegment)) {
+ returnType = ReturnType.COUNT;
+ queryDirection = Direction.BOTH;
+ countOnly = true;
+ } else if (IN_COUNT.equals(directionSegment)) {
+ returnType = ReturnType.COUNT;
+ queryDirection = Direction.IN;
+ countOnly = true;
+ } else if (OUT_COUNT.equals(directionSegment)) {
+ returnType = ReturnType.COUNT;
+ queryDirection = Direction.OUT;
+ countOnly = true;
+ } else if (BOTH_IDS.equals(directionSegment)) {
+ returnType = ReturnType.VERTEX_IDS;
+ queryDirection = Direction.BOTH;
+ countOnly = false;
+ } else if (IN_IDS.equals(directionSegment)) {
+ returnType = ReturnType.VERTEX_IDS;
+ queryDirection = Direction.IN;
+ countOnly = false;
+ } else if (OUT_IDS.equals(directionSegment)) {
+ returnType = ReturnType.VERTEX_IDS;
+ queryDirection = Direction.OUT;
+ countOnly = false;
+ } else {
+ throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
+ .entity(Servlets.escapeJsonString(directionSegment + " segment was invalid."))
+ .build());
+ }
+ }
+
+ public Direction getQueryDirection() {
+ return queryDirection;
+ }
+
+ public ReturnType getReturnType() {
+ return returnType;
+ }
+
+ public boolean isCountOnly() {
+ return countOnly;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
new file mode 100755
index 0000000..ff65580
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import java.util.List;
+
+/**
+ * This class provides RESTful API for Types.
+ *
+ * A type is the description of any representable item;
+ * e.g. a Hive table
+ *
+ * You could represent any meta model representing any domain using these types.
+ */
+@Path("types")
+@Singleton
+public class TypesResource {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
+
+ private final MetadataService metadataService;
+
+ static final String TYPE_ALL = "all";
+
+ @Inject
+ public TypesResource(MetadataService metadataService) {
+ this.metadataService = metadataService;
+ }
+
+ /**
+ * Submits a type definition corresponding to a given type representing a meta model of a
+ * domain. Could represent things like Hive Database, Hive Table, etc.
+ */
+ @POST
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response submit(@Context HttpServletRequest request) {
+ try {
+ final String typeDefinition = Servlets.getRequestPayload(request);
+ LOG.debug("Creating type with definition {} ", typeDefinition);
+
+ JSONObject typesJson = metadataService.createType(typeDefinition);
+ final JSONArray typesJsonArray = typesJson.getJSONArray(MetadataServiceClient.TYPES);
+
+ JSONArray typesResponse = new JSONArray();
+ for (int i = 0; i < typesJsonArray.length(); i++) {
+ final String name = typesJsonArray.getString(i);
+ typesResponse.put(new JSONObject() {{
+ put(MetadataServiceClient.NAME, name);
+ }});
+ }
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(MetadataServiceClient.TYPES, typesResponse);
+ return Response.status(ClientResponse.Status.CREATED).entity(response).build();
+ } catch (MetadataException | IllegalArgumentException e) {
+ LOG.error("Unable to persist types", e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to persist types", e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Fetch the complete definition of a given type name which is unique.
+ *
+ * @param typeName name of a type which is unique.
+ */
+ @GET
+ @Path("{typeName}")
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response getDefinition(@Context HttpServletRequest request,
+ @PathParam("typeName") String typeName) {
+ try {
+ final String typeDefinition = metadataService.getTypeDefinition(typeName);
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.TYPENAME, typeName);
+ response.put(MetadataServiceClient.DEFINITION, typeDefinition);
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+
+ return Response.ok(response).build();
+ } catch (MetadataException e) {
+ LOG.error("Unable to get type definition for type {}", typeName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (JSONException | IllegalArgumentException e) {
+ LOG.error("Unable to get type definition for type {}", typeName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get type definition for type {}", typeName, e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Gets the list of trait type names registered in the type system.
+ *
+ * @param type type should be the name of enum
+ * org.apache.atlas.typesystem.types.DataTypes.TypeCategory
+ * Typically, would be one of all, TRAIT, CLASS, ENUM, STRUCT
+ * @return entity names response payload as json
+ */
+ @GET
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response getTypesByFilter(@Context HttpServletRequest request,
+ @DefaultValue(TYPE_ALL) @QueryParam("type") String type) {
+ try {
+ List<String> result;
+ if (TYPE_ALL.equals(type)) {
+ result = metadataService.getTypeNamesList();
+ } else {
+ DataTypes.TypeCategory typeCategory = DataTypes.TypeCategory.valueOf(type);
+ result = metadataService.getTypeNamesByCategory(typeCategory);
+ }
+
+ JSONObject response = new JSONObject();
+ response.put(MetadataServiceClient.RESULTS, new JSONArray(result));
+ response.put(MetadataServiceClient.COUNT, result.size());
+ response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
+
+ return Response.ok(response).build();
+ } catch (IllegalArgumentException | MetadataException ie) {
+ LOG.error("Unsupported typeName while retrieving type list {}", type);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse("Unsupported type " + type, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to get types list", e);
+ throw new WebApplicationException(
+ Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
new file mode 100755
index 0000000..48285f0
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.service;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.bio.SocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.IOException;
+
+/**
+ * This class embeds a Jetty server and a connector.
+ */
+public class EmbeddedServer {
+ private static final int DEFAULT_BUFFER_SIZE = 16192;
+
+ protected final Server server = new Server();
+
+ public EmbeddedServer(int port, String path) throws IOException {
+ Connector connector = getConnector(port);
+ server.addConnector(connector);
+
+ WebAppContext application = new WebAppContext(path, "/");
+ server.setHandler(application);
+ }
+
+ public static EmbeddedServer newServer(int port, String path, boolean secure) throws IOException {
+ if (secure) {
+ return new SecureEmbeddedServer(port, path);
+ } else {
+ return new EmbeddedServer(port, path);
+ }
+ }
+
+ protected Connector getConnector(int port) throws IOException {
+ Connector connector = new SocketConnector();
+ connector.setPort(port);
+ connector.setHost("0.0.0.0");
+
+ // this is to enable large header sizes when Kerberos is enabled with AD
+ final Integer bufferSize = getBufferSize();
+ connector.setHeaderBufferSize(bufferSize);
+ connector.setRequestBufferSize(bufferSize);
+
+ return connector;
+ }
+
+ private Integer getBufferSize() {
+ try {
+ PropertiesConfiguration configuration = new PropertiesConfiguration(
+ "application.properties");
+ return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE);
+ } catch (ConfigurationException e) {
+ // do nothing
+ }
+
+ return DEFAULT_BUFFER_SIZE;
+ }
+
+ public void start() throws Exception {
+ server.start();
+ server.join();
+ }
+
+ public void stop() throws Exception {
+ server.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
new file mode 100755
index 0000000..19e7783
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.service;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
+import static org.apache.atlas.security.SecurityProperties.CLIENT_AUTH_KEY;
+import static org.apache.atlas.security.SecurityProperties.DEFATULT_TRUSTORE_FILE_LOCATION;
+import static org.apache.atlas.security.SecurityProperties.DEFAULT_KEYSTORE_FILE_LOCATION;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
+
+/**
+ * This is a jetty server which requires client auth via certificates.
+ */
+public class SecureEmbeddedServer extends EmbeddedServer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SecureEmbeddedServer.class);
+
+ public SecureEmbeddedServer(int port, String path) throws IOException {
+ super(port, path);
+ }
+
+ protected Connector getConnector(int port) throws IOException {
+ PropertiesConfiguration config = getConfiguration();
+
+ SslSocketConnector connector = new SslSocketConnector();
+ connector.setPort(port);
+ connector.setHost("0.0.0.0");
+ connector.setKeystore(config.getString(KEYSTORE_FILE_KEY,
+ System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION)));
+ connector.setKeyPassword(getPassword(config, KEYSTORE_PASSWORD_KEY));
+ connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY,
+ System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION)));
+ connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY));
+ connector.setPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY));
+ connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY)));
+ return connector;
+ }
+
+ /**
+ * Retrieves a password from a configured credential provider or prompts for the password and stores it in the
+ * configured credential provider.
+ * @param config application configuration
+ * @param key the key/alias for the password.
+ * @return the password.
+ * @throws IOException
+ */
+ private String getPassword(PropertiesConfiguration config, String key) throws IOException {
+
+ String password;
+
+ String provider = config.getString(CERT_STORES_CREDENTIAL_PROVIDER_PATH);
+ if (provider != null) {
+ LOG.info("Attempting to retrieve password from configured credential provider path");
+ Configuration c = new Configuration();
+ c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
+ CredentialProvider credentialProvider =
+ CredentialProviderFactory.getProviders(c).get(0);
+ CredentialProvider.CredentialEntry entry = credentialProvider.getCredentialEntry(key);
+ if (entry == null) {
+ throw new IOException(String.format("No credential entry found for %s. " +
+ "Please create an entry in the configured credential provider", key));
+ } else {
+ password = String.valueOf(entry.getCredential());
+ }
+
+ } else {
+ throw new IOException("No credential provider path configured for storage of certificate store passwords");
+ }
+
+ return password;
+ }
+
+ /**
+ * Returns the application configuration.
+ * @return
+ */
+ protected PropertiesConfiguration getConfiguration() {
+ try {
+ return new PropertiesConfiguration("application.properties");
+ } catch (ConfigurationException e) {
+ throw new RuntimeException("Unable to load configuration: application.properties");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/util/DateTimeHelper.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/util/DateTimeHelper.java b/webapp/src/main/java/org/apache/atlas/web/util/DateTimeHelper.java
new file mode 100755
index 0000000..7dced7e
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/util/DateTimeHelper.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.util;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Support function to parse and format date.
+ */
+public final class DateTimeHelper {
+
+ public static final String ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm'Z'";
+ private static final String DATE_PATTERN =
+ "(2\\d\\d\\d|19\\d\\d)-(0[1-9]|1[012])-(0[1-9]|1[0-9]|2[0-9]|3[01])T" +
+ "([0-1][0-9]|2[0-3]):([0-5][0-9])Z";
+ private static final Pattern PATTERN = Pattern.compile(DATE_PATTERN);
+
+ private DateTimeHelper() {
+ }
+
+ public static String getTimeZoneId(TimeZone tz) {
+ return tz.getID();
+ }
+
+ public static DateFormat getDateFormat() {
+ DateFormat dateFormat = new SimpleDateFormat(ISO8601_FORMAT);
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return dateFormat;
+ }
+
+ public static String formatDateUTC(Date date) {
+ return (date != null) ? getDateFormat().format(date) : null;
+ }
+
+ public static Date parseDateUTC(String dateStr) {
+ if (!validate(dateStr)) {
+ throw new IllegalArgumentException(dateStr + " is not a valid UTC string");
+ }
+ try {
+ return getDateFormat().parse(dateStr);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static String formatDateUTCToISO8601(final String dateString,
+ final String dateStringFormat) {
+
+ try {
+ DateFormat dateFormat = new SimpleDateFormat(
+ dateStringFormat.substring(0, dateString.length()));
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return DateTimeHelper.formatDateUTC(dateFormat.parse(dateString));
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Validate date format with regular expression.
+ *
+ * @param date date address for validation
+ * @return true valid date fromat, false invalid date format
+ */
+ public static boolean validate(final String date) {
+
+ Matcher matcher = PATTERN.matcher(date);
+
+ if (matcher.matches()) {
+
+ matcher.reset();
+
+ if (matcher.find()) {
+
+ int year = Integer.parseInt(matcher.group(1));
+ String month = matcher.group(2);
+ String day = matcher.group(3);
+
+ if (day.equals("31")
+ && (month.equals("4") || month.equals("6")
+ || month.equals("9") || month.equals("11")
+ || month.equals("04") || month.equals("06") || month.equals("09"))) {
+ return false; // only 1,3,5,7,8,10,12 has 31 days
+ } else if (month.equals("2") || month.equals("02")) {
+ // leap year
+ if (year % 4 == 0) {
+ return !(day.equals("30") || day.equals("31"));
+ } else {
+ return !(day.equals("29") || day.equals("30") || day.equals("31"));
+ }
+ } else {
+ return true;
+ }
+ } else {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java b/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
new file mode 100755
index 0000000..996ead2
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.util;
+
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.ParamChecker;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+/**
+ * Utility functions for dealing with servlets.
+ */
+public final class Servlets {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Servlets.class);
+ private Servlets() {
+ /* singleton */
+ }
+
+ public static final String JSON_MEDIA_TYPE = MediaType.APPLICATION_JSON + "; charset=UTF-8";
+
+ /**
+ * Returns the user of the given request.
+ *
+ * @param httpRequest an HTTP servlet request
+ * @return the user
+ */
+ public static String getUserFromRequest(HttpServletRequest httpRequest) {
+ String user = httpRequest.getRemoteUser();
+ if (!StringUtils.isEmpty(user)) {
+ return user;
+ }
+
+ user = httpRequest.getParameter("user.name"); // available in query-param
+ if (!StringUtils.isEmpty(user)) {
+ return user;
+ }
+
+ user = httpRequest.getHeader("Remote-User"); // backwards-compatibility
+ if (!StringUtils.isEmpty(user)) {
+ return user;
+ }
+
+ return null;
+ }
+
+ /**
+ * Returns the URI of the given request.
+ *
+ * @param httpRequest an HTTP servlet request
+ * @return the URI, including the query string
+ */
+ public static String getRequestURI(HttpServletRequest httpRequest) {
+ final StringBuilder url = new StringBuilder(100).append(httpRequest.getRequestURI());
+ if (httpRequest.getQueryString() != null) {
+ url.append('?').append(httpRequest.getQueryString());
+ }
+
+ return url.toString();
+ }
+
+ /**
+ * Returns the full URL of the given request.
+ *
+ * @param httpRequest an HTTP servlet request
+ * @return the full URL, including the query string
+ */
+ public static String getRequestURL(HttpServletRequest httpRequest) {
+ final StringBuilder url = new StringBuilder(100).append(httpRequest.getRequestURL());
+ if (httpRequest.getQueryString() != null) {
+ url.append('?').append(httpRequest.getQueryString());
+ }
+
+ return url.toString();
+ }
+
+ public static Response getErrorResponse(Throwable e, Response.Status status) {
+ Response response = getErrorResponse(e.getMessage(), status);
+ JSONObject responseJson = (JSONObject) response.getEntity();
+ try {
+ responseJson.put(MetadataServiceClient.STACKTRACE, printStackTrace(e));
+ } catch (JSONException e1) {
+ LOG.warn("Could not construct error Json rensponse", e1);
+ }
+ return response;
+ }
+
+ private static String printStackTrace(Throwable t) {
+ StringWriter sw = new StringWriter();
+ t.printStackTrace(new PrintWriter(sw));
+ return sw.toString();
+ }
+
+ public static Response getErrorResponse(String message, Response.Status status) {
+ JSONObject errorJson = new JSONObject();
+ Object errorEntity = Servlets.escapeJsonString(message);
+ try {
+ errorJson.put(MetadataServiceClient.ERROR, errorEntity);
+ errorEntity = errorJson;
+ } catch (JSONException jsonE) {
+ LOG.warn("Could not construct error Json rensponse", jsonE);
+ }
+ return Response
+ .status(status)
+ .entity(errorEntity)
+ .type(JSON_MEDIA_TYPE)
+ .build();
+ }
+
+ public static String getRequestPayload(HttpServletRequest request) throws IOException {
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(request.getInputStream(), writer);
+ return writer.toString();
+ }
+
+ public static String getRequestId() {
+ return Thread.currentThread().getName();
+ }
+
+ public static String escapeJsonString(String inputStr) {
+ ParamChecker.notNull(inputStr, "Input String cannot be null");
+ return StringEscapeUtils.escapeJson(inputStr);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/Main.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/Main.java b/webapp/src/main/java/org/apache/hadoop/metadata/Main.java
deleted file mode 100755
index e778d45..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/Main.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metadata.web.service.EmbeddedServer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-
-/**
- * Driver for running Metadata as a standalone server with embedded jetty server.
- */
-public final class Main {
- private static final Logger LOG = LoggerFactory.getLogger(Main.class);
- private static final String APP_PATH = "app";
- private static final String APP_PORT = "port";
- private static final String METADATA_HOME = "metadata.home";
- private static final String METADATA_LOG_DIR = "metadata.log.dir";
-
- /**
- * Prevent users from constructing this.
- */
- private Main() {
- }
-
- private static CommandLine parseArgs(String[] args) throws ParseException {
- Options options = new Options();
- Option opt;
-
- opt = new Option(APP_PATH, true, "Application Path");
- opt.setRequired(false);
- options.addOption(opt);
-
- opt = new Option(APP_PORT, true, "Application Port");
- opt.setRequired(false);
- options.addOption(opt);
-
- return new GnuParser().parse(options, args);
- }
-
- public static void main(String[] args) throws Exception {
- CommandLine cmd = parseArgs(args);
- PropertiesConfiguration buildConfiguration =
- new PropertiesConfiguration("metadata-buildinfo.properties");
- String appPath = "webapp/target/metadata-webapp-" + getProjectVersion(buildConfiguration);
-
- if (cmd.hasOption(APP_PATH)) {
- appPath = cmd.getOptionValue(APP_PATH);
- }
-
- setApplicationHome();
- PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
- final String enableTLSFlag = configuration.getString("metadata.enableTLS");
- final int appPort = getApplicationPort(cmd, enableTLSFlag);
- final boolean enableTLS = isTLSEnabled(enableTLSFlag, appPort);
- configuration.setProperty("metadata.enableTLS", String.valueOf(enableTLS));
-
- showStartupInfo(buildConfiguration, enableTLS, appPort);
- EmbeddedServer server = EmbeddedServer.newServer(appPort, appPath, enableTLS);
- server.start();
- }
-
- private static void setApplicationHome() {
- if (System.getProperty(METADATA_HOME) == null) {
- System.setProperty(METADATA_HOME, "target");
- }
- if (System.getProperty(METADATA_LOG_DIR) == null) {
- System.setProperty(METADATA_LOG_DIR, "target/logs");
- }
- }
-
- public static String getProjectVersion(PropertiesConfiguration buildConfiguration) {
- return buildConfiguration.getString("project.version");
- }
-
- private static int getApplicationPort(CommandLine cmd, String enableTLSFlag) {
- final int appPort;
- if (cmd.hasOption(APP_PORT)) {
- appPort = Integer.valueOf(cmd.getOptionValue(APP_PORT));
- } else {
- // default : metadata.enableTLS is true
- appPort = StringUtils.isEmpty(enableTLSFlag)
- || enableTLSFlag.equals("true") ? 21443 : 21000;
- }
-
- return appPort;
- }
-
- private static boolean isTLSEnabled(String enableTLSFlag, int appPort) {
- return Boolean.valueOf(StringUtils.isEmpty(enableTLSFlag)
- ? System
- .getProperty("metadata.enableTLS", (appPort % 1000) == 443 ? "true" : "false")
- : enableTLSFlag);
- }
-
- private static void showStartupInfo(PropertiesConfiguration buildConfiguration,
- boolean enableTLS, int appPort) {
- StringBuilder buffer = new StringBuilder();
- buffer.append("\n############################################");
- buffer.append("############################################");
- buffer.append("\n DGI Server (STARTUP)");
- buffer.append("\n");
- try {
- final Iterator<String> keys = buildConfiguration.getKeys();
- while (keys.hasNext()) {
- String key = keys.next();
- buffer.append('\n').append('\t').append(key).
- append(":\t").append(buildConfiguration.getProperty(key));
- }
- } catch (Throwable e) {
- buffer.append("*** Unable to get build info ***");
- }
- buffer.append("\n############################################");
- buffer.append("############################################");
- LOG.info(buffer.toString());
- LOG.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
- LOG.info("Server starting with TLS ? {} on port {}", enableTLS, appPort);
- LOG.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/examples/QuickStart.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/examples/QuickStart.java b/webapp/src/main/java/org/apache/hadoop/metadata/examples/QuickStart.java
deleted file mode 100755
index bf1ef39..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/examples/QuickStart.java
+++ /dev/null
@@ -1,478 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.examples;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.util.List;
-
-/**
- * A driver that sets up sample types and data for testing purposes.
- * Please take a look at QueryDSL in docs for the Meta Model.
- * todo - move this to examples module.
- */
-public class QuickStart {
-
- public static void main(String[] args) throws Exception {
- String baseUrl = getServerUrl(args);
- QuickStart quickStart = new QuickStart(baseUrl);
-
- // Shows how to create types in DGI for your meta model
- quickStart.createTypes();
-
- // Shows how to create entities (instances) for the added types in DGI
- quickStart.createEntities();
-
- // Shows some search queries using DSL based on types
- quickStart.search();
- }
-
- static String getServerUrl(String[] args) {
- String baseUrl = "http://localhost:21000";
- if (args.length > 0) {
- baseUrl = args[0];
- }
-
- return baseUrl;
- }
-
- private static final String DATABASE_TYPE = "DB";
- private static final String COLUMN_TYPE = "Column";
- private static final String TABLE_TYPE = "Table";
- private static final String VIEW_TYPE = "View";
- private static final String LOAD_PROCESS_TYPE = "LoadProcess";
- private static final String STORAGE_DESC_TYPE = "StorageDesc";
-
- private static final String[] TYPES = {
- DATABASE_TYPE, TABLE_TYPE, STORAGE_DESC_TYPE, COLUMN_TYPE, LOAD_PROCESS_TYPE, VIEW_TYPE,
- "JdbcAccess", "ETL", "Metric", "PII", "Fact", "Dimension"
- };
-
- private final MetadataServiceClient metadataServiceClient;
-
- QuickStart(String baseUrl) {
- metadataServiceClient = new MetadataServiceClient(baseUrl);
- }
-
- void createTypes() throws Exception {
- TypesDef typesDef = createTypeDefinitions();
-
- String typesAsJSON = TypesSerialization.toJson(typesDef);
- System.out.println("typesAsJSON = " + typesAsJSON);
- metadataServiceClient.createType(typesAsJSON);
-
- // verify types created
- verifyTypesCreated();
- }
-
- TypesDef createTypeDefinitions() throws Exception {
- HierarchicalTypeDefinition<ClassType> dbClsDef
- = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("description", DataTypes.STRING_TYPE),
- attrDef("locationUri", DataTypes.STRING_TYPE),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> storageDescClsDef =
- TypesUtil.createClassTypeDef(STORAGE_DESC_TYPE, null,
- attrDef("location", DataTypes.STRING_TYPE),
- attrDef("inputFormat", DataTypes.STRING_TYPE),
- attrDef("outputFormat", DataTypes.STRING_TYPE),
- attrDef("compressed", DataTypes.STRING_TYPE,
- Multiplicity.REQUIRED, false, null)
- );
-
- HierarchicalTypeDefinition<ClassType> columnClsDef =
- TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("dataType", DataTypes.STRING_TYPE),
- attrDef("comment", DataTypes.STRING_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> tblClsDef =
- TypesUtil.createClassTypeDef(TABLE_TYPE, ImmutableList.of("DataSet"),
- new AttributeDefinition("db", DATABASE_TYPE,
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("sd", STORAGE_DESC_TYPE,
- Multiplicity.REQUIRED, true, null),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE),
- attrDef("lastAccessTime", DataTypes.INT_TYPE),
- attrDef("retention", DataTypes.INT_TYPE),
- attrDef("viewOriginalText", DataTypes.STRING_TYPE),
- attrDef("viewExpandedText", DataTypes.STRING_TYPE),
- attrDef("tableType", DataTypes.STRING_TYPE),
- attrDef("temporary", DataTypes.BOOLEAN_TYPE),
- new AttributeDefinition("columns",
- DataTypes.arrayTypeName(COLUMN_TYPE),
- Multiplicity.COLLECTION, true, null)
- );
-
- HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
- TypesUtil.createClassTypeDef(LOAD_PROCESS_TYPE, ImmutableList.of("Process"),
- attrDef("userName", DataTypes.STRING_TYPE),
- attrDef("startTime", DataTypes.INT_TYPE),
- attrDef("endTime", DataTypes.INT_TYPE),
- attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
- );
-
- HierarchicalTypeDefinition<ClassType> viewClsDef =
- TypesUtil.createClassTypeDef(VIEW_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("db", DATABASE_TYPE,
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("inputTables",
- DataTypes.arrayTypeName(TABLE_TYPE),
- Multiplicity.COLLECTION, false, null)
- );
-
- HierarchicalTypeDefinition<TraitType> dimTraitDef =
- TypesUtil.createTraitTypeDef("Dimension", null);
-
- HierarchicalTypeDefinition<TraitType> factTraitDef =
- TypesUtil.createTraitTypeDef("Fact", null);
-
- HierarchicalTypeDefinition<TraitType> piiTraitDef =
- TypesUtil.createTraitTypeDef("PII", null);
-
- HierarchicalTypeDefinition<TraitType> metricTraitDef =
- TypesUtil.createTraitTypeDef("Metric", null);
-
- HierarchicalTypeDefinition<TraitType> etlTraitDef =
- TypesUtil.createTraitTypeDef("ETL", null);
-
- HierarchicalTypeDefinition<TraitType> jdbcTraitDef =
- TypesUtil.createTraitTypeDef("JdbcAccess", null);
-
- return TypeUtils.getTypesDef(
- ImmutableList.<EnumTypeDefinition>of(),
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(dimTraitDef, factTraitDef,
- piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
- ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef,
- tblClsDef, loadProcessClsDef, viewClsDef)
- );
- }
-
- AttributeDefinition attrDef(String name, IDataType dT) {
- return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
- return attrDef(name, dT, m, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT,
- Multiplicity m, boolean isComposite, String reverseAttributeName) {
- Preconditions.checkNotNull(name);
- Preconditions.checkNotNull(dT);
- return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
- }
-
- void createEntities() throws Exception {
- Id salesDB = database(
- "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
-
-
- Referenceable sd = rawStorageDescriptor("hdfs://host:8000/apps/warehouse/sales",
- "TextInputFormat", "TextOutputFormat", true);
-
- List<Referenceable> salesFactColumns = ImmutableList.of(
- rawColumn("time_id", "int", "time id"),
- rawColumn("product_id", "int", "product id"),
- rawColumn("customer_id", "int", "customer id", "PII"),
- rawColumn("sales", "double", "product id", "Metric")
- );
-
- Id salesFact = table("sales_fact", "sales fact table",
- salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
-
- List<Referenceable> productDimColumns = ImmutableList.of(
- rawColumn("product_id", "int", "product id"),
- rawColumn("product_name", "string", "product name"),
- rawColumn("brand_name", "int", "brand name")
- );
-
- Id productDim = table("product_dim", "product dimension table",
- salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
-
- List<Referenceable> timeDimColumns = ImmutableList.of(
- rawColumn("time_id", "int", "time id"),
- rawColumn("dayOfYear", "int", "day Of Year"),
- rawColumn("weekDay", "int", "week Day")
- );
-
- Id timeDim = table("time_dim", "time dimension table",
- salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
-
-
- List<Referenceable> customerDimColumns = ImmutableList.of(
- rawColumn("customer_id", "int", "customer id", "PII"),
- rawColumn("name", "string", "customer name", "PII"),
- rawColumn("address", "string", "customer address", "PII")
- );
-
- Id customerDim = table("customer_dim", "customer dimension table",
- salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
-
-
- Id reportingDB = database("Reporting", "reporting database", "Jane BI",
- "hdfs://host:8000/apps/warehouse/reporting");
-
- Id salesFactDaily = table("sales_fact_daily_mv",
- "sales fact daily materialized view", reportingDB, sd,
- "Joe BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL",
- ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
- "create table as select ", "plan", "id", "graph",
- "ETL");
-
- view("product_dim_view", reportingDB,
- ImmutableList.of(productDim), "Dimension", "JdbcAccess");
-
- view("customer_dim_view", reportingDB,
- ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
-
- Id salesFactMonthly = table("sales_fact_monthly_mv",
- "sales fact monthly materialized view",
- reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL",
- ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
- "create table as select ", "plan", "id", "graph",
- "ETL");
- }
-
- private Id createInstance(Referenceable referenceable) throws Exception {
- String typeName = referenceable.getTypeName();
-
- String entityJSON = InstanceSerialization.toJson(referenceable, true);
- System.out.println("Submitting new entity= " + entityJSON);
- JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
- String guid = jsonObject.getString(MetadataServiceClient.GUID);
- System.out.println("created instance for type " + typeName + ", guid: " + guid);
-
- // return the Id for created instance with guid
- return new Id(guid, referenceable.getId().getVersion(), referenceable.getTypeName());
- }
-
- Id database(String name, String description,
- String owner, String locationUri,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("locationUri", locationUri);
- referenceable.set("createTime", System.currentTimeMillis());
-
- return createInstance(referenceable);
- }
-
- Referenceable rawStorageDescriptor(String location, String inputFormat,
- String outputFormat,
- boolean compressed) throws Exception {
- Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
- referenceable.set("location", location);
- referenceable.set("inputFormat", inputFormat);
- referenceable.set("outputFormat", outputFormat);
- referenceable.set("compressed", compressed);
-
- return referenceable;
- }
-
- Referenceable rawColumn(String name, String dataType, String comment,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("dataType", dataType);
- referenceable.set("comment", comment);
-
- return referenceable;
- }
-
- Id table(String name, String description,
- Id dbId, Referenceable sd,
- String owner, String tableType,
- List<Referenceable> columns,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(TABLE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("tableType", tableType);
- referenceable.set("createTime", System.currentTimeMillis());
- referenceable.set("lastAccessTime", System.currentTimeMillis());
- referenceable.set("retention", System.currentTimeMillis());
- referenceable.set("db", dbId);
- referenceable.set("sd", sd);
- referenceable.set("columns", columns);
-
- return createInstance(referenceable);
- }
-
- Id loadProcess(String name, String description, String user,
- List<Id> inputTables,
- List<Id> outputTables,
- String queryText, String queryPlan,
- String queryId, String queryGraph,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(LOAD_PROCESS_TYPE, traitNames);
- // super type attributes
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("inputs", inputTables);
- referenceable.set("outputs", outputTables);
-
- referenceable.set("user", user);
- referenceable.set("startTime", System.currentTimeMillis());
- referenceable.set("endTime", System.currentTimeMillis() + 10000);
-
- referenceable.set("queryText", queryText);
- referenceable.set("queryPlan", queryPlan);
- referenceable.set("queryId", queryId);
- referenceable.set("queryGraph", queryGraph);
-
- return createInstance(referenceable);
- }
-
- Id view(String name, Id dbId,
- List<Id> inputTables,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("db", dbId);
-
- referenceable.set("inputTables", inputTables);
-
- return createInstance(referenceable);
- }
-
- private void verifyTypesCreated() throws Exception {
- List<String> types = metadataServiceClient.listTypes();
- for (String type : TYPES) {
- assert types.contains(type);
- }
- }
-
- private String[] getDSLQueries() {
- return new String[]{
- "from DB",
- "DB",
- "DB where name=\"Reporting\"",
- "DB where DB.name=\"Reporting\"",
- "DB name = \"Reporting\"",
- "DB DB.name = \"Reporting\"",
- "DB where name=\"Reporting\" select name, owner",
- "DB where DB.name=\"Reporting\" select name, owner",
- "DB has name",
- "DB where DB has name",
- "DB, Table",
- "DB is JdbcAccess",
- /*
- "DB, hive_process has name",
- "DB as db1, Table where db1.name = \"Reporting\"",
- "DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
- */
- "from Table",
- "Table",
- "Table is Dimension",
- "Column where Column isa PII",
- "View is Dimension",
- /*"Column where Column isa PII select Column.name",*/
- "Column select Column.name",
- "Column select name",
- "Column where Column.name=\"customer_id\"",
- "from Table select Table.name",
- "DB where (name = \"Reporting\")",
- "DB where (name = \"Reporting\") select name as _col_0, owner as _col_1",
- "DB where DB is JdbcAccess",
- "DB where DB has name",
- "DB Table",
- "DB where DB has name",
- "DB as db1 Table where (db1.name = \"Reporting\")",
- "DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ",
- /*
- todo: does not work
- "DB where (name = \"Reporting\") and ((createTime + 1) > 0)",
- "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
- "DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
- "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
- "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
- */
- // trait searches
- "Dimension",
- /*"Fact", - todo: does not work*/
- "JdbcAccess",
- "ETL",
- "Metric",
- "PII",
- /*
- // Lineage - todo - fix this, its not working
- "Table hive_process outputTables",
- "Table loop (hive_process outputTables)",
- "Table as _loop0 loop (hive_process outputTables) withPath",
- "Table as src loop (hive_process outputTables) as dest select src.name as srcTable, dest.name as destTable withPath",
- */
- "Table where name=\"sales_fact\", columns",
- "Table where name=\"sales_fact\", columns as column select column.name, column.dataType, column.comment",
- "from DataSet",
- "from Process",
- };
- }
-
- private void search() throws Exception {
- for (String dslQuery : getDSLQueries()) {
- JSONObject response = metadataServiceClient.searchEntity(dslQuery);
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- if (!results.isNull("rows")) {
- JSONArray rows = results.getJSONArray("rows");
- System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
- } else {
- System.out.println("query [" + dslQuery + "] failed, results:" + results.toString());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/util/CredentialProviderUtility.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/util/CredentialProviderUtility.java b/webapp/src/main/java/org/apache/hadoop/metadata/util/CredentialProviderUtility.java
deleted file mode 100755
index a91db75..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/util/CredentialProviderUtility.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.util;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-
-import java.io.*;
-import java.util.Arrays;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-/**
- * A utility class for generating a credential provider containing the entries required for supporting the SSL implementation
- * of the DGC server.
- */
-public class CredentialProviderUtility {
- private static final String[] KEYS = new String[] {KEYSTORE_PASSWORD_KEY,
- TRUSTSTORE_PASSWORD_KEY, SERVER_CERT_PASSWORD_KEY};
-
- public static abstract class TextDevice {
- public abstract void printf(String fmt, Object... params);
-
- public abstract String readLine(String fmt, Object ... args);
-
- public abstract char[] readPassword(String fmt, Object ... args);
-
- }
-
- private static TextDevice DEFAULT_TEXT_DEVICE = new TextDevice() {
- Console console = System.console();
-
- @Override
- public void printf(String fmt, Object... params) {
- console.printf(fmt, params);
- }
-
- @Override
- public String readLine(String fmt, Object ... args) {
- return console.readLine(fmt, args);
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- return console.readPassword(fmt, args);
- }
- };
-
- public static TextDevice textDevice = DEFAULT_TEXT_DEVICE;
-
- public static void main(String[] args) throws IOException {
- // prompt for the provider name
- CredentialProvider provider = getCredentialProvider(textDevice);
-
- char[] cred;
- for (String key : KEYS) {
- cred = getPassword(textDevice, key);
- // create a credential entry and store it
- boolean overwrite = true;
- if (provider.getCredentialEntry(key) != null) {
- String choice = textDevice.readLine("Entry for %s already exists. Overwrite? (y/n) [y]:", key);
- overwrite = StringUtils.isEmpty(choice) || choice.equalsIgnoreCase("y");
- if (overwrite) {
- provider.deleteCredentialEntry(key);
- provider.flush();
- provider.createCredentialEntry(key, cred);
- provider.flush();
- textDevice.printf("Entry for %s was overwritten with the new value.\n", key);
- } else {
- textDevice.printf("Entry for %s was not overwritten.\n", key);
- }
- } else {
- provider.createCredentialEntry(key, cred);
- provider.flush();
- }
- }
- }
-
- /**
- * Retrieves a password from the command line.
- * @param textDevice the system console.
- * @param key the password key/alias.
- * @return the password.
- */
- private static char[] getPassword(TextDevice textDevice, String key) {
- boolean noMatch;
- char[] cred = new char[0];
- char[] passwd1;
- char[] passwd2;
- do {
- passwd1 = textDevice.readPassword("Please enter the password value for %s:", key);
- passwd2 = textDevice.readPassword("Please enter the password value for %s again:", key);
- noMatch = !Arrays.equals(passwd1, passwd2);
- if (noMatch) {
- if (passwd1 != null) Arrays.fill(passwd1, ' ');
- textDevice.printf("Password entries don't match. Please try again.\n");
- } else {
- if (passwd1.length == 0) {
- textDevice.printf("An empty password is not valid. Please try again.\n");
- noMatch = true;
- } else {
- cred = passwd1;
- }
- }
- if (passwd2 != null) Arrays.fill(passwd2, ' ');
- } while (noMatch);
- return cred;
- }
-
- /**\
- * Returns a credential provider for the entered JKS path.
- * @param textDevice the system console.
- * @return the Credential provider
- * @throws IOException
- */
- private static CredentialProvider getCredentialProvider(TextDevice textDevice) throws IOException {
- String providerPath = textDevice.readLine("Please enter the full path to the credential provider:");
- File file = new File(providerPath);
- if (file.exists()) {
- textDevice.printf("%s already exists. You will need to specify whether existing entries should be overwritten " +
- "(default is 'yes')\n", providerPath);
- }
- String providerURI = JavaKeyStoreProvider.SCHEME_NAME + "://file" + providerPath;
- Configuration conf = new Configuration(false);
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerURI);
- return CredentialProviderFactory.getProviders(conf).get(0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/errors/LoggingExceptionMapper.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/errors/LoggingExceptionMapper.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/errors/LoggingExceptionMapper.java
deleted file mode 100755
index f94b4cf..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/errors/LoggingExceptionMapper.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.errors;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.ExceptionMapper;
-import java.util.concurrent.ThreadLocalRandom;
-
-/**
- * Exception mapper for Jersey.
- * @param <E>
- */
-public class LoggingExceptionMapper<E extends Throwable> implements ExceptionMapper<E> {
- private static final Logger LOGGER = LoggerFactory.getLogger(LoggingExceptionMapper.class);
-
- @Override
- public Response toResponse(E exception) {
- if (exception instanceof WebApplicationException) {
- return ((WebApplicationException) exception).getResponse();
- }
-
- final long id = ThreadLocalRandom.current().nextLong();
- logException(id, exception);
- return Response.serverError()
- .entity(formatErrorMessage(id, exception))
- .build();
- }
-
- @SuppressWarnings("UnusedParameters")
- protected String formatErrorMessage(long id, E exception) {
- return String.format(
- "There was an error processing your request. It has been logged (ID %016x).", id);
- }
-
- protected void logException(long id, E exception) {
- LOGGER.error(formatLogMessage(id, exception), exception);
- }
-
- @SuppressWarnings("UnusedParameters")
- protected String formatLogMessage(long id, Throwable exception) {
- return String.format("Error handling a request: %016x", id);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/AuditFilter.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/AuditFilter.java
deleted file mode 100755
index 3ee86d5..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/AuditFilter.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.filters;
-
-import com.google.inject.Singleton;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.web.util.DateTimeHelper;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.util.Date;
-import java.util.UUID;
-
-/**
- * This records audit information as part of the filter after processing the request
- * and also introduces a UUID into request and response for tracing requests in logs.
- */
-@Singleton
-public class AuditFilter implements Filter {
-
- private static final Logger AUDIT_LOG = LoggerFactory.getLogger("AUDIT");
- private static final Logger LOG = LoggerFactory.getLogger(AuditFilter.class);
-
- @Override
- public void init(FilterConfig filterConfig) throws ServletException {
- LOG.info("AuditFilter initialization started");
- }
-
- @Override
- public void doFilter(ServletRequest request,
- ServletResponse response,
- FilterChain filterChain) throws IOException, ServletException {
- final String requestTimeISO9601 = DateTimeHelper.formatDateUTC(new Date());
- final HttpServletRequest httpRequest = (HttpServletRequest) request;
- final String requestId = UUID.randomUUID().toString();
- final Thread currentThread = Thread.currentThread();
- final String oldName = currentThread.getName();
-
- try {
- currentThread.setName(formatName(oldName, requestId));
- recordAudit(httpRequest, requestTimeISO9601);
- filterChain.doFilter(request, response);
- } finally {
- // put the request id into the response so users can trace logs for this request
- ((HttpServletResponse) response).setHeader(MetadataServiceClient.REQUEST_ID, requestId);
- currentThread.setName(oldName);
- }
- }
-
- private String formatName(String oldName, String requestId) {
- return oldName + " - " + requestId;
- }
-
- private void recordAudit(HttpServletRequest httpRequest, String whenISO9601) {
- final String who = getUserFromRequest(httpRequest);
- final String fromHost = httpRequest.getRemoteHost();
- final String fromAddress = httpRequest.getRemoteAddr();
- final String whatURL = Servlets.getRequestURL(httpRequest);
- final String whatAddrs = httpRequest.getLocalAddr();
-
- LOG.debug("Audit: {}/{} performed request {} ({}) at time {}",
- who, fromAddress, whatURL, whatAddrs, whenISO9601);
- audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
- }
-
- private String getUserFromRequest(HttpServletRequest httpRequest) {
- // look for the user in the request
- final String userFromRequest = Servlets.getUserFromRequest(httpRequest);
- return userFromRequest == null ? "UNKNOWN" : userFromRequest;
- }
-
- private void audit(String who, String fromAddress, String fromHost, String whatURL,
- String whatAddrs, String whenISO9601) {
- AUDIT_LOG.info("Audit: {}/{}-{} performed request {} ({}) at time {}",
- who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
- }
-
- @Override
- public void destroy() {
- // do nothing
- }
-}
\ No newline at end of file
[50/58] [abbrv] incubator-atlas git commit: HDPDGI-76:Baban:Hide tabs
other than details is not Table
Posted by ve...@apache.org.
HDPDGI-76:Baban:Hide tabs other than details is not Table
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/d54a5981
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/d54a5981
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/d54a5981
Branch: refs/heads/master
Commit: d54a5981f6bb40cc992ec71b52f841621e26c5c2
Parents: 0e18270
Author: Vishal Kadam <vi...@gmail.com>
Authored: Tue Jun 16 00:02:19 2015 -0400
Committer: Vishal Kadam <vi...@gmail.com>
Committed: Tue Jun 16 00:08:08 2015 -0400
----------------------------------------------------------------------
dashboard/v2/bower.json | 1 -
dashboard/v2/public/modules/details/detailsController.js | 8 ++++----
dashboard/v2/public/modules/details/views/details.html | 6 +++---
3 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d54a5981/dashboard/v2/bower.json
----------------------------------------------------------------------
diff --git a/dashboard/v2/bower.json b/dashboard/v2/bower.json
index 711be46..581f682 100755
--- a/dashboard/v2/bower.json
+++ b/dashboard/v2/bower.json
@@ -1,4 +1,3 @@
-
{
"name": "dgc-metadata",
"description": "Apache Atlas",
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d54a5981/dashboard/v2/public/modules/details/detailsController.js
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/modules/details/detailsController.js b/dashboard/v2/public/modules/details/detailsController.js
index b5550a3..0e96d42 100644
--- a/dashboard/v2/public/modules/details/detailsController.js
+++ b/dashboard/v2/public/modules/details/detailsController.js
@@ -22,19 +22,19 @@ angular.module('dgc.details').controller('DetailsController', ['$window', '$scop
function($window, $scope, $stateParams, DetailsResource) {
$scope.tableName = false;
+ $scope.isTable = false;
+
DetailsResource.get({
id: $stateParams.id
}, function(data) {
$scope.details = data;
+ $scope.schemas = data;
$scope.tableName = data.values.name;
+ $scope.isTable = data.typeName === 'Table';
});
$scope.isString = angular.isString;
- $scope.schemas = DetailsResource.get({
- id: $stateParams.id
- });
-
$scope.onActivate = function tabActivate(tabname) {
$scope.$broadcast('render-lineage', {
type: tabname,
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/d54a5981/dashboard/v2/public/modules/details/views/details.html
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/modules/details/views/details.html b/dashboard/v2/public/modules/details/views/details.html
index 5e3cae5..19f0ce0 100644
--- a/dashboard/v2/public/modules/details/views/details.html
+++ b/dashboard/v2/public/modules/details/views/details.html
@@ -41,9 +41,9 @@
</tbody>
</table>
</tab>
- <tab data-heading="Schema"><ng-include src="'/modules/details/views/schema.html'"/></tab>
- <tab data-heading="Output" data-disable="!tableName" data-select="onActivate('outputs')"><ng-include data-table-type="outputs" src="'/modules/lineage/views/lineage.html'"/></tab>
- <tab data-heading="Input" data-disable="!tableName" data-select="onActivate('inputs')"><ng-include data-table-type="inputs" src="'/modules/lineage/views/lineage.html'"/></tab>
+ <tab data-heading="Schema" data-ng-if="isTable"><ng-include src="'/modules/details/views/schema.html'"/></tab>
+ <tab data-heading="Output" data-ng-if="isTable" data-disable="!tableName" data-select="onActivate('outputs')"><ng-include data-table-type="outputs" src="'/modules/lineage/views/lineage.html'"/></tab>
+ <tab data-heading="Input" data-ng-if="isTable" data-disable="!tableName" data-select="onActivate('inputs')"><ng-include data-table-type="inputs" src="'/modules/lineage/views/lineage.html'"/></tab>
</tabset>
</div>
</div>
[35/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
new file mode 100755
index 0000000..bcfc938
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -0,0 +1,1292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.common.base.Preconditions;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanProperty;
+import com.thinkaurelius.titan.core.TitanVertex;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.GraphQuery;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.GraphTransaction;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.MapIds;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * An implementation backed by a Graph database provided
+ * as a Graph Service.
+ */
+@Singleton
+public class GraphBackedMetadataRepository implements MetadataRepository {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
+ private static final String FULL_TEXT_DELIMITER = " ";
+
+ private static final String EDGE_LABEL_PREFIX = "__";
+
+ private final AtomicInteger ID_SEQ = new AtomicInteger(0);
+
+ private final TypedInstanceToGraphMapper instanceToGraphMapper
+ = new TypedInstanceToGraphMapper();
+ private final GraphToTypedInstanceMapper graphToInstanceMapper
+ = new GraphToTypedInstanceMapper();
+
+ private final TypeSystem typeSystem;
+ private final TitanGraph titanGraph;
+
+ @Inject
+ public GraphBackedMetadataRepository(
+ GraphProvider<TitanGraph> graphProvider) throws MetadataException {
+ this.typeSystem = TypeSystem.getInstance();
+
+ this.titanGraph = graphProvider.get();
+ }
+
+ public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
+ return graphToInstanceMapper;
+ }
+
+ @Override
+ public String getTypeAttributeName() {
+ return Constants.ENTITY_TYPE_PROPERTY_KEY;
+ }
+
+ /**
+ * Returns the property key used to store super type names.
+ *
+ * @return property key used to store super type names.
+ */
+ @Override
+ public String getSuperTypeAttributeName() {
+ return Constants.SUPER_TYPES_PROPERTY_KEY;
+ }
+
+ public String getIdAttributeName() {
+ return Constants.GUID_PROPERTY_KEY;
+ }
+
+ @Override
+ public String getTraitLabel(IDataType<?> dataType, String traitName) {
+ return dataType.getName() + "." + traitName;
+ }
+
+ @Override
+ public String getFieldNameInVertex(IDataType<?> dataType,
+ AttributeInfo aInfo) throws MetadataException {
+ return getQualifiedName(dataType, aInfo.name);
+ }
+
+ @Override
+ public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
+ return getEdgeLabel(dataType.getName(), aInfo.name);
+ }
+
+ public String getEdgeLabel(String typeName, String attrName) {
+ return EDGE_LABEL_PREFIX + typeName + "." + attrName;
+ }
+
+ public String getEdgeLabel(ITypedInstance typedInstance,
+ AttributeInfo aInfo) throws MetadataException {
+ IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
+ return getEdgeLabel(dataType, aInfo);
+ }
+
+ @Override
+ @GraphTransaction
+ public String createEntity(IReferenceableInstance typedInstance) throws RepositoryException {
+ LOG.info("adding entity={}", typedInstance);
+ try {
+ return instanceToGraphMapper.mapTypedInstanceToGraph(typedInstance);
+ } catch (MetadataException e) {
+ throw new RepositoryException(e);
+ }
+ }
+
+ @Override
+ @GraphTransaction
+ public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException {
+ LOG.info("Retrieving entity with guid={}", guid);
+
+ Vertex instanceVertex = getVertexForGUID(guid);
+
+ try {
+ LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
+ return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
+
+ } catch (MetadataException e) {
+ throw new RepositoryException(e);
+ }
+ }
+
+ private Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
+ Vertex instanceVertex = GraphHelper.findVertexByGUID(titanGraph, guid);
+ if (instanceVertex == null) {
+ LOG.debug("Could not find a vertex for guid={}", guid);
+ throw new EntityNotFoundException(
+ "Could not find an entity in the repository for guid: " + guid);
+ }
+
+ return instanceVertex;
+ }
+
+ @Override
+ public List<String> getEntityList(String entityType) throws RepositoryException {
+ LOG.info("Retrieving entity list for type={}", entityType);
+ GraphQuery query = titanGraph.query()
+ .has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
+ Iterator<Vertex> results = query.vertices().iterator();
+ if (!results.hasNext()) {
+ return Collections.emptyList();
+ }
+
+ ArrayList<String> entityList = new ArrayList<>();
+ while (results.hasNext()) {
+ Vertex vertex = results.next();
+ entityList.add(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY));
+ }
+
+ return entityList;
+ }
+
+ /**
+ * Gets the list of trait names for a given entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return a list of trait names for the given entity guid
+ * @throws RepositoryException
+ */
+ @Override
+ @GraphTransaction
+ public List<String> getTraitNames(String guid) throws MetadataException {
+ LOG.info("Retrieving trait names for entity={}", guid);
+ Vertex instanceVertex = getVertexForGUID(guid);
+ return getTraitNames(instanceVertex);
+ }
+
+ public List<String> getTraitNames(Vertex entityVertex) {
+ ArrayList<String> traits = new ArrayList<>();
+ for (TitanProperty property : ((TitanVertex) entityVertex)
+ .getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
+ traits.add((String) property.getValue());
+ }
+
+ return traits;
+ }
+
+ /**
+ * Adds a new trait to an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitInstance trait instance that needs to be added to entity
+ * @throws RepositoryException
+ */
+ @Override
+ @GraphTransaction
+ public void addTrait(String guid,
+ ITypedStruct traitInstance) throws RepositoryException {
+ Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
+ final String traitName = traitInstance.getTypeName();
+ LOG.info("Adding a new trait={} for entity={}", traitName, guid);
+
+ try {
+ Vertex instanceVertex = getVertexForGUID(guid);
+
+ // add the trait instance as a new vertex
+ final String typeName = getTypeName(instanceVertex);
+ instanceToGraphMapper.mapTraitInstanceToVertex(
+ traitInstance, getIdFromVertex(typeName, instanceVertex),
+ typeName, instanceVertex, Collections.<Id, Vertex>emptyMap());
+
+ // update the traits in entity once adding trait instance is successful
+ ((TitanVertex) instanceVertex)
+ .addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
+
+ } catch (Exception e) {
+ throw new RepositoryException(e);
+ }
+ }
+
+ /**
+ * Deletes a given trait from an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitNameToBeDeleted name of the trait
+ * @throws RepositoryException
+ */
+ @Override
+ @GraphTransaction
+ public void deleteTrait(String guid, String traitNameToBeDeleted)
+ throws RepositoryException {
+ LOG.info("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
+ try {
+ Vertex instanceVertex = getVertexForGUID(guid);
+
+ List<String> traitNames = getTraitNames(instanceVertex);
+ if (!traitNames.contains(traitNameToBeDeleted)) {
+ throw new RepositoryException("Could not find trait=" + traitNameToBeDeleted
+ + " in the repository for entity: " + guid);
+ }
+
+ final String entityTypeName = getTypeName(instanceVertex);
+ String relationshipLabel = getEdgeLabel(entityTypeName, traitNameToBeDeleted);
+ Iterator<Edge> results = instanceVertex.getEdges(
+ Direction.OUT, relationshipLabel).iterator();
+ if (results.hasNext()) { // there should only be one edge for this label
+ final Edge traitEdge = results.next();
+ final Vertex traitVertex = traitEdge.getVertex(Direction.IN);
+
+ // remove the edge to the trait instance from the repository
+ titanGraph.removeEdge(traitEdge);
+
+ if (traitVertex != null) { // remove the trait instance from the repository
+ titanGraph.removeVertex(traitVertex);
+
+ // update the traits in entity once trait removal is successful
+ traitNames.remove(traitNameToBeDeleted);
+ updateTraits(instanceVertex, traitNames);
+ }
+ }
+ } catch (Exception e) {
+ throw new RepositoryException(e);
+ }
+ }
+
+ private void updateTraits(Vertex instanceVertex, List<String> traitNames) {
+ // remove the key
+ instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
+
+ // add it back again
+ for (String traitName : traitNames) {
+ ((TitanVertex) instanceVertex).addProperty(
+ Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
+ }
+ }
+
+ @Override
+ @GraphTransaction
+ public void updateEntity(String guid,
+ String property, String value) throws RepositoryException {
+ LOG.info("Adding property {} for entity guid {}", property, guid);
+
+ try {
+ Vertex instanceVertex = getVertexForGUID(guid);
+
+ LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
+ String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ ClassType type = typeSystem.getDataType(ClassType.class, typeName);
+ AttributeInfo attributeInfo = type.fieldMapping.fields.get(property);
+ if (attributeInfo == null) {
+ throw new MetadataException("Invalid property "
+ + property + " for entity " + typeName);
+ }
+
+ DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
+ ITypedReferenceableInstance instance = type.createInstance();
+ if (attrTypeCategory == DataTypes.TypeCategory.PRIMITIVE) {
+ instance.set(property, value);
+ } else if (attrTypeCategory == DataTypes.TypeCategory.CLASS) {
+ Id id = new Id(value, 0, attributeInfo.dataType().getName());
+ instance.set(property, id);
+ } else {
+ throw new RepositoryException("Update of " + attrTypeCategory + " is not supported");
+ }
+
+ instanceToGraphMapper.mapAttributesToVertex(getIdFromVertex(typeName, instanceVertex),
+ instance, instanceVertex, new HashMap<Id, Vertex>(),
+ attributeInfo, attributeInfo.dataType());
+ } catch (Exception e) {
+ throw new RepositoryException(e);
+ }
+ }
+
+ public Id getIdFromVertex(String dataTypeName, Vertex vertex) {
+ return new Id(
+ vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
+ vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ dataTypeName);
+ }
+
+ String getTypeName(Vertex instanceVertex) {
+ return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ }
+
+
+ String getQualifiedName(ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws MetadataException {
+ IDataType dataType = typeSystem.getDataType(
+ IDataType.class, typedInstance.getTypeName());
+ return getQualifiedName(dataType, attributeInfo.name);
+ }
+
+ String getQualifiedName(IDataType dataType,
+ String attributeName) throws MetadataException {
+ return dataType.getTypeCategory() == DataTypes.TypeCategory.STRUCT
+ ? dataType.getName() + "." + attributeName
+ // else class or trait
+ : ((HierarchicalType) dataType).getQualifiedName(attributeName);
+ }
+
+ private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
+
+ public final Map<Id, Id> idToNewIdMap;
+ public final Map<Id, IReferenceableInstance> idToInstanceMap;
+ public final Map<Id, Vertex> idToVertexMap;
+
+ public EntityProcessor() {
+ idToNewIdMap = new HashMap<>();
+ idToInstanceMap = new HashMap<>();
+ idToVertexMap = new HashMap<>();
+ }
+
+ @Override
+ public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
+ IReferenceableInstance ref = null;
+ Id id = null;
+
+ if (nd.attributeName == null) {
+ ref = (IReferenceableInstance) nd.instance;
+ id = ref.getId();
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ if (nd.value != null && (nd.value instanceof Id)) {
+ id = (Id) nd.value;
+ }
+ }
+
+ if (id != null) {
+ if (id.isUnassigned()) {
+ if (!idToNewIdMap.containsKey(id)) {
+ idToNewIdMap.put(id, new Id(ID_SEQ.getAndIncrement(), 0, id.className));
+ }
+
+ if (ref != null) {
+ if (idToInstanceMap.containsKey(id)) { // Oops
+ throw new RepositoryException(String.format(
+ "Unexpected internal error: Id %s processed again", id));
+ }
+
+ idToInstanceMap.put(id, ref);
+ }
+ }
+ }
+ }
+
+ public void createVerticesForClassTypes(
+ List<ITypedReferenceableInstance> newInstances) throws MetadataException {
+ for (ITypedReferenceableInstance typedInstance : newInstances) {
+ final Id id = typedInstance.getId();
+ if (!idToVertexMap.containsKey(id)) {
+ Vertex instanceVertex;
+ if (id.isAssigned()) { // has a GUID
+ instanceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
+ } else {
+ ClassType classType = typeSystem.getDataType(
+ ClassType.class, typedInstance.getTypeName());
+ instanceVertex = GraphHelper.createVertexWithIdentity(
+ titanGraph, typedInstance,
+ classType.getAllSuperTypeNames());
+ }
+
+ idToVertexMap.put(id, instanceVertex);
+ }
+ }
+ }
+ }
+
+ private final class TypedInstanceToGraphMapper {
+
+ private String mapTypedInstanceToGraph(IReferenceableInstance typedInstance)
+ throws MetadataException {
+
+ EntityProcessor entityProcessor = new EntityProcessor();
+ try {
+ LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
+ new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
+ } catch (MetadataException me) {
+ throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
+ }
+
+ List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor);
+ entityProcessor.createVerticesForClassTypes(newTypedInstances);
+ String guid = addDiscoveredInstances(typedInstance, entityProcessor, newTypedInstances);
+ addFullTextProperty(entityProcessor, newTypedInstances);
+ return guid;
+ }
+
+ private void addFullTextProperty(EntityProcessor entityProcessor,
+ List<ITypedReferenceableInstance> newTypedInstances)
+ throws MetadataException {
+
+ for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse
+ Id id = typedInstance.getId();
+ Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
+ String fullText = getFullTextForVertex(instanceVertex, true);
+ addProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
+ }
+ }
+
+ private String getFullTextForVertex(Vertex instanceVertex,
+ boolean followReferences) throws MetadataException {
+ String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ ITypedReferenceableInstance typedReference =
+ graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
+ String fullText = getFullTextForInstance(typedReference, followReferences);
+ StringBuilder fullTextBuilder = new StringBuilder(
+ typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
+
+ List<String> traits = typedReference.getTraits();
+ for (String traitName : traits) {
+ String traitText = getFullTextForInstance(
+ (ITypedInstance) typedReference.getTrait(traitName), false);
+ fullTextBuilder.append(FULL_TEXT_DELIMITER)
+ .append(traitName)
+ .append(FULL_TEXT_DELIMITER)
+ .append(traitText);
+ }
+ return fullTextBuilder.toString();
+ }
+
+ private String getFullTextForAttribute(IDataType type, Object value,
+ boolean followReferences) throws MetadataException {
+ switch (type.getTypeCategory()) {
+ case PRIMITIVE:
+ return String.valueOf(value);
+
+ case ENUM:
+ return ((EnumValue)value).value;
+
+ case ARRAY:
+ StringBuilder fullText = new StringBuilder();
+ IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
+ List list = (List) value;
+
+ for (Object element : list) {
+ String elemFullText = getFullTextForAttribute(elemType, element, false);
+ if (StringUtils.isNotEmpty(elemFullText))
+ fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
+ }
+ return fullText.toString();
+
+ case MAP:
+ fullText = new StringBuilder();
+ IDataType keyType = ((DataTypes.MapType) type).getKeyType();
+ IDataType valueType = ((DataTypes.MapType) type).getValueType();
+ Map map = (Map) value;
+
+ for (Object entryObj : map.entrySet()) {
+ Map.Entry entry = (Map.Entry) entryObj;
+ String keyFullText = getFullTextForAttribute(keyType, entry.getKey(), false);
+ if (StringUtils.isNotEmpty(keyFullText)) {
+ fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
+ }
+ String valueFullText = getFullTextForAttribute(valueType, entry.getValue(), false);
+ if (StringUtils.isNotEmpty(valueFullText)) {
+ fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
+ }
+ }
+ return fullText.toString();
+
+ case CLASS:
+ if (followReferences) {
+ String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
+ Vertex refVertex = getVertexForGUID(refGuid);
+ return getFullTextForVertex(refVertex, false);
+ }
+ break;
+
+ case STRUCT:
+ if (followReferences) {
+ return getFullTextForInstance((ITypedInstance) value, false);
+ }
+ break;
+
+ default:
+ throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
+
+ }
+ return null;
+ }
+
+ private String getFullTextForInstance(ITypedInstance typedInstance,
+ boolean followReferences) throws MetadataException {
+ StringBuilder fullText = new StringBuilder();
+ for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ if (attrValue == null) {
+ continue;
+ }
+
+ String attrFullText = getFullTextForAttribute(
+ attributeInfo.dataType(), attrValue, followReferences);
+ if (StringUtils.isNotEmpty(attrFullText)) {
+ fullText = fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name)
+ .append(FULL_TEXT_DELIMITER).append(attrFullText);
+ }
+ }
+ return fullText.toString();
+ }
+
+ /**
+ * Step 2: Traverse oldIdToInstance map create newInstances :
+ * List[ITypedReferenceableInstance]
+ * - create a ITypedReferenceableInstance.
+ * replace any old References ( ids or object references) with new Ids.
+ */
+ private List<ITypedReferenceableInstance> discoverInstances(EntityProcessor entityProcessor)
+ throws RepositoryException {
+ List<ITypedReferenceableInstance> newTypedInstances = new ArrayList<>();
+ for (IReferenceableInstance transientInstance : entityProcessor.idToInstanceMap.values()) {
+ LOG.debug("Discovered instance {}", transientInstance.getTypeName());
+ try {
+ ClassType cT = typeSystem.getDataType(
+ ClassType.class, transientInstance.getTypeName());
+ ITypedReferenceableInstance newInstance = cT.convert(
+ transientInstance, Multiplicity.REQUIRED);
+ newTypedInstances.add(newInstance);
+
+ // Now replace old references with new Ids
+ MapIds mapIds = new MapIds(entityProcessor.idToNewIdMap);
+ new ObjectGraphWalker(typeSystem, mapIds, newTypedInstances).walk();
+
+ } catch (MetadataException me) {
+ throw new RepositoryException(
+ String.format("Failed to create Instance(id = %s",
+ transientInstance.getId()), me);
+ }
+ }
+
+ return newTypedInstances;
+ }
+
+ private String addDiscoveredInstances(IReferenceableInstance entity,
+ EntityProcessor entityProcessor,
+ List<ITypedReferenceableInstance> newTypedInstances)
+ throws MetadataException {
+
+ String typedInstanceGUID = null;
+ for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse over newInstances
+ LOG.debug("Adding typed instance {}", typedInstance.getTypeName());
+
+ Id id = typedInstance.getId();
+ if (id == null) { // oops
+ throw new RepositoryException("id cannot be null");
+ }
+
+ Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
+
+ // add the attributes for the instance
+ ClassType classType = typeSystem.getDataType(
+ ClassType.class, typedInstance.getTypeName());
+ final Map<String, AttributeInfo> fields = classType.fieldMapping().fields;
+
+ mapInstanceToVertex(
+ id, typedInstance, instanceVertex, fields, entityProcessor.idToVertexMap);
+
+ for (String traitName : typedInstance.getTraits()) {
+ LOG.debug("mapping trait {}", traitName);
+ ((TitanVertex) instanceVertex)
+ .addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
+ ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
+
+ // add the attributes for the trait instance
+ mapTraitInstanceToVertex(traitInstance, typedInstance,
+ instanceVertex, entityProcessor.idToVertexMap);
+ }
+
+ if (typedInstance.getId() == entity.getId()) { // save the guid for return
+ typedInstanceGUID = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ }
+ }
+
+ return typedInstanceGUID;
+ }
+
+ private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
+ Map<String, AttributeInfo> fields,
+ Map<Id, Vertex> idToVertexMap) throws MetadataException {
+ LOG.debug("Mapping instance {} of {} to vertex {}",
+ typedInstance, typedInstance.getTypeName(), instanceVertex);
+ for (AttributeInfo attributeInfo : fields.values()) {
+ final IDataType dataType = attributeInfo.dataType();
+ mapAttributesToVertex(id, typedInstance, instanceVertex,
+ idToVertexMap, attributeInfo, dataType);
+ }
+ }
+
+ private void mapAttributesToVertex(Id id, ITypedInstance typedInstance,
+ Vertex instanceVertex,
+ Map<Id, Vertex> idToVertexMap,
+ AttributeInfo attributeInfo,
+ IDataType dataType) throws MetadataException {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
+ final String propertyName = getQualifiedName(typedInstance, attributeInfo);
+ String edgeLabel = getEdgeLabel(typedInstance, attributeInfo);
+ if (attrValue == null) {
+ return;
+ }
+
+ switch (dataType.getTypeCategory()) {
+ case PRIMITIVE:
+ mapPrimitiveToVertex(typedInstance, instanceVertex, attributeInfo);
+ break;
+
+ case ENUM:
+ //handles both int and string for enum
+ EnumValue enumValue = (EnumValue) dataType.convert(typedInstance.get(attributeInfo.name),
+ Multiplicity.REQUIRED);
+ addProperty(instanceVertex, propertyName, enumValue.value);
+ break;
+
+ case ARRAY:
+ mapArrayCollectionToVertex(
+ id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
+ break;
+
+ case MAP:
+ mapMapCollectionToVertex(
+ id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
+ break;
+
+ case STRUCT:
+ Vertex structInstanceVertex = mapStructInstanceToVertex(id,
+ (ITypedStruct) typedInstance.get(attributeInfo.name), attributeInfo, idToVertexMap);
+ // add an edge to the newly created vertex from the parent
+ GraphHelper.addEdge(
+ titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
+ break;
+
+ case TRAIT:
+ // do NOTHING - this is taken care of earlier
+ break;
+
+ case CLASS:
+ Id referenceId = (Id) typedInstance.get(attributeInfo.name);
+ mapClassReferenceAsEdge(
+ instanceVertex, idToVertexMap, edgeLabel, referenceId);
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown type category: "
+ + dataType.getTypeCategory());
+ }
+ }
+
+ private void mapArrayCollectionToVertex(Id id, ITypedInstance typedInstance,
+ Vertex instanceVertex,
+ AttributeInfo attributeInfo,
+ Map<Id, Vertex> idToVertexMap)
+ throws MetadataException {
+ LOG.debug("Mapping instance {} to vertex {} for name {}",
+ typedInstance.getTypeName(), instanceVertex, attributeInfo.name);
+ List list = (List) typedInstance.get(attributeInfo.name);
+ if (list == null || list.isEmpty()) {
+ return;
+ }
+
+ String propertyName = getQualifiedName(typedInstance, attributeInfo);
+ IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
+
+ List<String> values = new ArrayList<>(list.size());
+ for (int index = 0; index < list.size(); index++) {
+ String entryId = mapCollectionEntryToVertex(id, instanceVertex,
+ attributeInfo, idToVertexMap, elementType, list.get(index), propertyName);
+ values.add(entryId);
+ }
+
+ // for dereference on way out
+ addProperty(instanceVertex, propertyName, values);
+ }
+
+ private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance,
+ Vertex instanceVertex,
+ AttributeInfo attributeInfo,
+ Map<Id, Vertex> idToVertexMap)
+ throws MetadataException {
+ LOG.debug("Mapping instance {} to vertex {} for name {}",
+ typedInstance.getTypeName(), instanceVertex, attributeInfo.name);
+ @SuppressWarnings("unchecked")
+ Map<Object, Object> collection =
+ (Map<Object, Object>) typedInstance.get(attributeInfo.name);
+ if (collection == null || collection.isEmpty()) {
+ return;
+ }
+
+ String propertyName = getQualifiedName(typedInstance, attributeInfo);
+ IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
+ for (Map.Entry entry : collection.entrySet()) {
+ String myPropertyName = propertyName + "." + entry.getKey().toString();
+ String value = mapCollectionEntryToVertex(id, instanceVertex, attributeInfo,
+ idToVertexMap, elementType, entry.getValue(), myPropertyName);
+ addProperty(instanceVertex, myPropertyName, value);
+ }
+
+ // for dereference on way out
+ addProperty(instanceVertex, propertyName, new ArrayList(collection.keySet()));
+ }
+
+ private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex,
+ AttributeInfo attributeInfo,
+ Map<Id, Vertex> idToVertexMap,
+ IDataType elementType, Object value,
+ String propertyName) throws MetadataException {
+ final String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
+ switch (elementType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ return value.toString();
+
+ case ARRAY:
+ case MAP:
+ case TRAIT:
+ // do nothing
+ return null;
+
+ case STRUCT:
+ Vertex structInstanceVertex = mapStructInstanceToVertex(id,
+ (ITypedStruct) value, attributeInfo, idToVertexMap);
+ // add an edge to the newly created vertex from the parent
+ Edge structElementEdge = GraphHelper.addEdge(
+ titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
+ return structElementEdge.getId().toString();
+
+ case CLASS:
+ Id referenceId = (Id) value;
+ return mapClassReferenceAsEdge(
+ instanceVertex, idToVertexMap, edgeLabel, referenceId);
+
+ default:
+ throw new IllegalArgumentException("Unknown type category: "
+ + elementType.getTypeCategory());
+ }
+ }
+
+ private String mapClassReferenceAsEdge(Vertex instanceVertex,
+ Map<Id, Vertex> idToVertexMap,
+ String propertyKey, Id id) throws MetadataException {
+ if (id != null) {
+ Vertex referenceVertex;
+ if (id.isAssigned()) {
+ referenceVertex = GraphHelper.findVertexByGUID(titanGraph, id.id);
+ } else {
+ referenceVertex = idToVertexMap.get(id);
+ }
+
+ if (referenceVertex != null) {
+ // add an edge to the class vertex from the instance
+ Edge edge = GraphHelper.addEdge(
+ titanGraph, instanceVertex, referenceVertex, propertyKey);
+ return String.valueOf(edge.getId());
+ }
+ }
+
+ return null;
+ }
+
+ private Vertex mapStructInstanceToVertex(Id id, ITypedStruct structInstance,
+ AttributeInfo attributeInfo,
+ Map<Id, Vertex> idToVertexMap)
+ throws MetadataException {
+ // add a new vertex for the struct or trait instance
+ Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity(
+ titanGraph, structInstance.getTypeName(), id,
+ Collections.<String>emptySet()); // no super types for struct type
+ LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name,
+ structInstance);
+
+ // map all the attributes to this newly created vertex
+ mapInstanceToVertex(id, structInstance, structInstanceVertex,
+ structInstance.fieldMapping().fields, idToVertexMap);
+
+ return structInstanceVertex;
+ }
+
+ private void mapTraitInstanceToVertex(ITypedStruct traitInstance,
+ ITypedReferenceableInstance typedInstance,
+ Vertex parentInstanceVertex,
+ Map<Id, Vertex> idToVertexMap)
+ throws MetadataException {
+ // add a new vertex for the struct or trait instance
+ mapTraitInstanceToVertex(traitInstance, typedInstance.getId(),
+ typedInstance.getTypeName(), parentInstanceVertex, idToVertexMap);
+ }
+
+ private void mapTraitInstanceToVertex(ITypedStruct traitInstance,
+ Id typedInstanceId, String typedInstanceTypeName,
+ Vertex parentInstanceVertex,
+ Map<Id, Vertex> idToVertexMap)
+ throws MetadataException {
+ // add a new vertex for the struct or trait instance
+ final String traitName = traitInstance.getTypeName();
+ Vertex traitInstanceVertex = GraphHelper.createVertexWithoutIdentity(
+ titanGraph, traitInstance.getTypeName(), typedInstanceId,
+ typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
+ LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
+
+ // map all the attributes to this newly created vertex
+ mapInstanceToVertex(typedInstanceId, traitInstance, traitInstanceVertex,
+ traitInstance.fieldMapping().fields, idToVertexMap);
+
+ // add an edge to the newly created vertex from the parent
+ String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName);
+ GraphHelper.addEdge(
+ titanGraph, parentInstanceVertex, traitInstanceVertex, relationshipLabel);
+ }
+
+ private void mapPrimitiveToVertex(ITypedInstance typedInstance,
+ Vertex instanceVertex,
+ AttributeInfo attributeInfo) throws MetadataException {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ if (attrValue == null) {
+ return; // add only if instance has this attribute
+ }
+
+ final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
+ Object propertyValue = null;
+ if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
+ propertyValue = typedInstance.getString(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
+ propertyValue = typedInstance.getShort(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
+ propertyValue = typedInstance.getInt(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ propertyValue = typedInstance.getBigInt(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
+ propertyValue = typedInstance.getBoolean(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
+ propertyValue = typedInstance.getByte(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
+ propertyValue = typedInstance.getLong(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
+ propertyValue = typedInstance.getFloat(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
+ propertyValue = typedInstance.getDouble(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+ propertyValue = typedInstance.getDate(attributeInfo.name);
+ }
+ addProperty(instanceVertex, vertexPropertyName, propertyValue);
+ }
+ }
+
+ private void addProperty(Vertex vertex, String propertyName, Object value) {
+ LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
+ vertex.setProperty(propertyName, value);
+ }
+
+ public final class GraphToTypedInstanceMapper {
+
+ public ITypedReferenceableInstance mapGraphToTypedInstance(String guid,
+ Vertex instanceVertex)
+ throws MetadataException {
+
+ LOG.debug("Mapping graph root vertex {} to typed instance for guid {}",
+ instanceVertex, guid);
+ String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ List<String> traits = getTraitNames(instanceVertex);
+
+ Id id = new Id(guid,
+ instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
+ LOG.debug("Created id {} for instance type {}", id, typeName);
+
+ ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
+ ITypedReferenceableInstance typedInstance = classType.createInstance(
+ id, traits.toArray(new String[traits.size()]));
+
+ mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
+ mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
+
+ return typedInstance;
+ }
+
+ public void mapVertexToInstanceTraits(Vertex instanceVertex,
+ ITypedReferenceableInstance typedInstance,
+ List<String> traits) throws MetadataException {
+ for (String traitName : traits) {
+ LOG.debug("mapping trait {} to instance", traitName);
+ TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
+ mapVertexToTraitInstance(
+ instanceVertex, typedInstance, traitName, traitType);
+ }
+ }
+
+ public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ Map<String, AttributeInfo> fields)
+ throws MetadataException {
+
+ LOG.debug("Mapping vertex {} to instance {} for fields",
+ instanceVertex, typedInstance.getTypeName(), fields);
+ for (AttributeInfo attributeInfo : fields.values()) {
+ mapVertexToAttribute(instanceVertex, typedInstance, attributeInfo);
+ }
+ }
+
+ public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws MetadataException {
+ LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
+ final IDataType dataType = attributeInfo.dataType();
+ final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
+
+ switch (dataType.getTypeCategory()) {
+ case PRIMITIVE:
+ mapVertexToPrimitive(instanceVertex, typedInstance, attributeInfo);
+ break; // add only if vertex has this attribute
+
+ case ENUM:
+ if (instanceVertex.getProperty(vertexPropertyName) == null) {
+ return;
+ }
+
+ typedInstance.set(attributeInfo.name, dataType.convert(instanceVertex.<String>getProperty
+ (vertexPropertyName), Multiplicity.REQUIRED));
+ break;
+
+ case ARRAY:
+ mapVertexToArrayInstance(
+ instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
+ break;
+
+ case MAP:
+ mapVertexToMapInstance(
+ instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
+ break;
+
+ case STRUCT:
+ mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
+ break;
+
+ case TRAIT:
+ // do NOTHING - handled in class
+ break;
+
+ case CLASS:
+ String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
+ Object idOrInstance = mapClassReferenceToVertex(instanceVertex,
+ attributeInfo, relationshipLabel, attributeInfo.dataType());
+ typedInstance.set(attributeInfo.name, idOrInstance);
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ public Object mapClassReferenceToVertex(Vertex instanceVertex,
+ AttributeInfo attributeInfo,
+ String relationshipLabel,
+ IDataType dataType) throws MetadataException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ Iterator<Edge> results = instanceVertex.getEdges(
+ Direction.OUT, relationshipLabel).iterator();
+ if (results.hasNext()) {
+ final Vertex referenceVertex = results.next().getVertex(Direction.IN);
+ if (referenceVertex != null) {
+ final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ LOG.debug("Found vertex {} for label {} with guid {}",
+ referenceVertex, relationshipLabel, guid);
+ if (attributeInfo.isComposite) {
+ LOG.debug("Found composite, mapping vertex to instance");
+ return mapGraphToTypedInstance(guid, referenceVertex);
+ } else {
+ Id referenceId = new Id(guid,
+ referenceVertex.<Integer>getProperty( Constants.VERSION_PROPERTY_KEY),
+ dataType.getName());
+ LOG.debug("Found non-composite, adding id {} ", referenceId);
+ return referenceId;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo,
+ String propertyName) throws MetadataException {
+ LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
+ List list = instanceVertex.getProperty(propertyName);
+ if (list == null || list.size() == 0) {
+ return;
+ }
+ DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
+ final IDataType elementType = arrayType.getElemType();
+
+ ArrayList values = new ArrayList();
+ for (Object listElement : list) {
+ values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, listElement,
+ propertyName));
+ }
+
+ typedInstance.set(attributeInfo.name, values);
+ }
+
+ public Object mapVertexToCollectionEntry(Vertex instanceVertex,
+ AttributeInfo attributeInfo,
+ IDataType elementType, Object value, String propertyName)
+ throws MetadataException {
+ String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
+ switch (elementType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ return value;
+
+ case ARRAY:
+ case MAP:
+ case TRAIT:
+ // do nothing
+ break;
+
+ case STRUCT:
+ return getStructInstanceFromVertex(instanceVertex,
+ elementType, attributeInfo.name, edgeLabel, (String) value);
+
+ case CLASS:
+ return mapClassReferenceToVertex(
+ instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
+
+ default:
+ break;
+ }
+
+ throw new IllegalArgumentException();
+ }
+
+ @SuppressWarnings("unchecked")
+ private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo,
+ final String propertyName) throws MetadataException {
+ LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
+ List<String> keys = instanceVertex.getProperty(propertyName);
+ if (keys == null || keys.size() == 0) {
+ return;
+ }
+ DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
+ final IDataType valueType = mapType.getValueType();
+
+ HashMap values = new HashMap();
+ for (String key : keys) {
+ String keyPropertyName = propertyName + "." + key;
+ Object keyValue = instanceVertex.getProperty(keyPropertyName);
+ values.put(key, mapVertexToCollectionEntry(instanceVertex, attributeInfo,
+ valueType, keyValue, propertyName));
+ }
+
+ typedInstance.set(attributeInfo.name, values);
+ }
+
+ private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex,
+ IDataType elemType,
+ String attributeName, String relationshipLabel,
+ String edgeId) throws MetadataException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ if (edgeId.equals(String.valueOf(edge.getId()))) {
+ Vertex structInstanceVertex = edge.getVertex(Direction.IN);
+ LOG.debug("mapping vertex {} to struct {}", structInstanceVertex,
+ attributeName);
+
+ if (structInstanceVertex != null) {
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ",
+ structInstanceVertex, elemType.getName());
+ StructType structType = typeSystem
+ .getDataType(StructType.class, elemType.getName());
+ ITypedStruct structInstance = structType.createInstance();
+
+ mapVertexToInstance(structInstanceVertex, structInstance,
+ structType.fieldMapping().fields);
+ return structInstance;
+ }
+
+ break;
+ }
+ }
+
+ return null;
+ }
+
+ public Object mapClassReferenceToVertex(Vertex instanceVertex,
+ AttributeInfo attributeInfo,
+ String relationshipLabel,
+ IDataType dataType,
+ String edgeId) throws MetadataException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ if (edgeId.equals(String.valueOf(edge.getId()))) {
+ final Vertex referenceVertex = edge.getVertex(Direction.IN);
+ if (referenceVertex != null) {
+ final String guid = referenceVertex
+ .getProperty(Constants.GUID_PROPERTY_KEY);
+ LOG.debug("Found vertex {} for label {} with guid {}",
+ referenceVertex, relationshipLabel, guid);
+ if (attributeInfo.isComposite) {
+ LOG.debug("Found composite, mapping vertex to instance");
+ return mapGraphToTypedInstance(guid, referenceVertex);
+ } else {
+ Id referenceId = new Id(guid,
+ referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ dataType.getName());
+ LOG.debug("Found non-composite, adding id {} ", referenceId);
+ return referenceId;
+ }
+ }
+
+ break;
+ }
+ }
+
+ return null;
+ }
+
+ private void mapVertexToStructInstance(Vertex instanceVertex,
+ ITypedInstance typedInstance,
+ AttributeInfo attributeInfo)
+ throws MetadataException {
+ LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
+ StructType structType = typeSystem.getDataType(
+ StructType.class, attributeInfo.dataType().getName());
+ ITypedStruct structInstance = structType.createInstance();
+ typedInstance.set(attributeInfo.name, structInstance);
+
+ String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
+ if (structInstanceVertex != null) {
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ",
+ structInstanceVertex, structInstance.getTypeName());
+ mapVertexToInstance(structInstanceVertex, structInstance,
+ structType.fieldMapping().fields);
+ break;
+ }
+ }
+ }
+
+ private void mapVertexToTraitInstance(Vertex instanceVertex,
+ ITypedReferenceableInstance typedInstance,
+ String traitName,
+ TraitType traitType) throws MetadataException {
+ ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
+
+ mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(),
+ traitName, traitType, traitInstance);
+ }
+
+ public void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName,
+ String traitName, TraitType traitType,
+ ITypedStruct traitInstance) throws MetadataException {
+ String relationshipLabel = getEdgeLabel(typedInstanceTypeName, traitName);
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
+ if (traitInstanceVertex != null) {
+ LOG.debug("Found trait instance vertex {}, mapping to instance {} ",
+ traitInstanceVertex, traitInstance.getTypeName());
+ mapVertexToInstance(traitInstanceVertex, traitInstance,
+ traitType.fieldMapping().fields);
+ break;
+ }
+ }
+ }
+
+ private void mapVertexToPrimitive(Vertex instanceVertex,
+ ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws MetadataException {
+ LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
+ final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
+ if (instanceVertex.getProperty(vertexPropertyName) == null) {
+ return;
+ }
+
+ if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
+ typedInstance.setString(attributeInfo.name,
+ instanceVertex.<String>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
+ typedInstance.setShort(attributeInfo.name,
+ instanceVertex.<Short>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
+ typedInstance.setInt(attributeInfo.name,
+ instanceVertex.<Integer>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ typedInstance.setBigInt(attributeInfo.name,
+ instanceVertex.<BigInteger>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
+ typedInstance.setBoolean(attributeInfo.name,
+ instanceVertex.<Boolean>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
+ typedInstance.setByte(attributeInfo.name,
+ instanceVertex.<Byte>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
+ typedInstance.setLong(attributeInfo.name,
+ instanceVertex.<Long>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
+ typedInstance.setFloat(attributeInfo.name,
+ instanceVertex.<Float>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
+ typedInstance.setDouble(attributeInfo.name,
+ instanceVertex.<Double>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ typedInstance.setBigDecimal(attributeInfo.name,
+ instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+ typedInstance.setDate(attributeInfo.name,
+ instanceVertex.<Date>getProperty(vertexPropertyName));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
new file mode 100755
index 0000000..0375c0b
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -0,0 +1,348 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.inject.Singleton;
+import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.PropertyKey;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.schema.Mapping;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
+import com.thinkaurelius.titan.core.schema.TitanManagement;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.discovery.SearchIndexer;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.Map;
+
+/**
+ * Adds index for properties of a given type when its added before any instances are added.
+ */
+@Singleton
+public class GraphBackedSearchIndexer implements SearchIndexer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
+
+ private final TitanGraph titanGraph;
+
+ @Inject
+ public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider)
+ throws RepositoryException {
+
+ this.titanGraph = graphProvider.get();
+
+ initialize();
+ }
+
+ /**
+ * Initializes the indices for the graph - create indices for Global Vertex Keys
+ */
+ private void initialize() {
+ TitanManagement management = titanGraph.getManagementSystem();
+ if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
+ LOG.info("Global indexes already exist for graph");
+ return;
+ }
+
+ LOG.info("Indexes do not exist, Creating indexes for titanGraph.");
+ management.buildIndex(Constants.VERTEX_INDEX, Vertex.class)
+ .buildMixedIndex(Constants.BACKING_INDEX);
+ management.buildIndex(Constants.EDGE_INDEX, Edge.class)
+ .buildMixedIndex(Constants.BACKING_INDEX);
+ management.commit();
+
+ // create a composite index for guid as its unique
+ createCompositeIndex(Constants.GUID_INDEX,
+ Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE);
+
+ // create a composite and mixed index for type since it can be combined with other keys
+ createCompositeAndMixedIndex(Constants.ENTITY_TYPE_INDEX,
+ Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE);
+
+ // create a composite and mixed index for type since it can be combined with other keys
+ createCompositeAndMixedIndex(Constants.SUPER_TYPES_INDEX,
+ Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET);
+
+ // create a composite and mixed index for traitNames since it can be combined with other
+ // keys. Traits must be a set and not a list.
+ createCompositeAndMixedIndex(Constants.TRAIT_NAMES_INDEX,
+ Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET);
+
+ // Index for full text search
+ createFullTextIndex();
+
+ //Indexes for graph backed type system store
+ createTypeStoreIndexes();
+
+ LOG.info("Index creation for global keys complete.");
+ }
+
+ private void createFullTextIndex() {
+ TitanManagement management = titanGraph.getManagementSystem();
+ PropertyKey fullText =
+ management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();
+
+ management.buildIndex(Constants.FULLTEXT_INDEX, Vertex.class)
+ .addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
+ .buildMixedIndex(Constants.BACKING_INDEX);
+ management.commit();
+ LOG.info("Created mixed index for {}", Constants.ENTITY_TEXT_PROPERTY_KEY);
+ }
+
+ private void createTypeStoreIndexes() {
+ //Create unique index on typeName
+ createCompositeIndex(Constants.TYPENAME_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY, String.class,
+ true, Cardinality.SINGLE);
+
+ //create index on vertex type
+ createCompositeIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class,
+ false, Cardinality.SINGLE);
+
+ }
+
+ /**
+ * This is upon adding a new type to Store.
+ *
+ * @param typeName type name
+ * @param dataType data type
+ * @throws org.apache.atlas.MetadataException
+ */
+ @Override
+ public void onAdd(String typeName, IDataType dataType) throws MetadataException {
+ LOG.info("Creating indexes for type name={}, definition={}", typeName, dataType);
+
+ try {
+ addIndexForType(dataType);
+ LOG.info("Index creation for type {} complete", typeName);
+
+ } catch (Throwable throwable) {
+ // gets handle to currently open transaction
+ titanGraph.getManagementSystem().rollback();
+ LOG.error("Error creating index for type {}", dataType, throwable);
+ }
+ }
+
+ private void addIndexForType(IDataType dataType) {
+ switch (dataType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ case ARRAY:
+ case MAP:
+ // do nothing since these are only attributes
+ // and not types like structs, traits or classes
+ break;
+
+ case STRUCT:
+ StructType structType = (StructType) dataType;
+ createIndexForFields(structType, structType.fieldMapping().fields);
+ break;
+
+ case TRAIT:
+ TraitType traitType = (TraitType) dataType;
+ createIndexForFields(traitType, traitType.fieldMapping().fields);
+ break;
+
+ case CLASS:
+ ClassType classType = (ClassType) dataType;
+ createIndexForFields(classType, classType.fieldMapping().fields);
+ break;
+
+ default:
+ throw new IllegalArgumentException("bad data type" + dataType);
+ }
+ }
+
+ private void createIndexForFields(IDataType dataType, Map<String, AttributeInfo> fields) {
+ for (AttributeInfo field : fields.values()) {
+ if (field.isIndexable) {
+ createIndexForAttribute(dataType.getName(), field);
+ }
+ }
+ }
+
+ private void createIndexForAttribute(String typeName, AttributeInfo field) {
+ final String propertyName = typeName + "." + field.name;
+ switch (field.dataType().getTypeCategory()) {
+ case PRIMITIVE:
+ createVertexMixedIndex(propertyName, getPrimitiveClass(field.dataType()));
+ break;
+
+ case ENUM:
+ createVertexMixedIndex(propertyName, String.class);
+ break;
+
+ case ARRAY:
+ case MAP:
+ // todo - how do we overcome this limitation?
+ // IGNORE: Can only index single-valued property keys on vertices in Mixed Index
+ break;
+
+ case STRUCT:
+ StructType structType = (StructType) field.dataType();
+ createIndexForFields(structType, structType.fieldMapping().fields);
+ break;
+
+ case TRAIT:
+ // do nothing since this is NOT contained in other types
+ break;
+
+ case CLASS:
+ // this is only A reference, index the attribute for edge
+ // Commenting this out since we do not need an index for edge here
+ //createEdgeMixedIndex(propertyName);
+ break;
+
+ default:
+ throw new IllegalArgumentException("bad data type" + field.dataType().getName());
+ }
+ }
+
+ private Class getPrimitiveClass(IDataType dataType) {
+ if (dataType == DataTypes.STRING_TYPE) {
+ return String.class;
+ } else if (dataType == DataTypes.SHORT_TYPE) {
+ return Short.class;
+ } else if (dataType == DataTypes.INT_TYPE) {
+ return Integer.class;
+ } else if (dataType == DataTypes.BIGINTEGER_TYPE) {
+ return BigInteger.class;
+ } else if (dataType == DataTypes.BOOLEAN_TYPE) {
+ return Boolean.class;
+ } else if (dataType == DataTypes.BYTE_TYPE) {
+ return Byte.class;
+ } else if (dataType == DataTypes.LONG_TYPE) {
+ return Long.class;
+ } else if (dataType == DataTypes.FLOAT_TYPE) {
+ return Float.class;
+ } else if (dataType == DataTypes.DOUBLE_TYPE) {
+ return Double.class;
+ } else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
+ return BigDecimal.class;
+ } else if (dataType == DataTypes.DATE_TYPE) {
+ return Date.class;
+ }
+
+
+ throw new IllegalArgumentException("unknown data type " + dataType);
+ }
+
+/*
+ private Cardinality getCardinality(Multiplicity multiplicity) {
+ if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
+ return Cardinality.SINGLE;
+ } else if (multiplicity == Multiplicity.COLLECTION) {
+ return Cardinality.LIST;
+ } else if (multiplicity == Multiplicity.SET) {
+ return Cardinality.SET;
+ }
+
+ // todo - default to LIST as this is the most forgiving
+ return Cardinality.LIST;
+ }
+*/
+
+ private void createCompositeAndMixedIndex(String indexName,
+ String propertyName, Class propertyClass,
+ boolean isUnique, Cardinality cardinality) {
+ createCompositeIndex(indexName, propertyName, propertyClass, isUnique, cardinality);
+ createVertexMixedIndex(propertyName, propertyClass);
+ }
+
+ private PropertyKey createCompositeIndex(String indexName,
+ String propertyName, Class propertyClass,
+ boolean isUnique, Cardinality cardinality) {
+ TitanManagement management = titanGraph.getManagementSystem();
+ PropertyKey propertyKey = management.getPropertyKey(propertyName);
+ if (propertyKey == null) {
+ propertyKey = management
+ .makePropertyKey(propertyName)
+ .dataType(propertyClass)
+ .cardinality(cardinality)
+ .make();
+
+ TitanManagement.IndexBuilder indexBuilder = management
+ .buildIndex(indexName, Vertex.class)
+ .addKey(propertyKey);
+
+ if (isUnique) {
+ indexBuilder = indexBuilder.unique();
+ }
+
+ indexBuilder.buildCompositeIndex();
+ management.commit();
+
+ LOG.info("Created index for property {} in composite index {}", propertyName, indexName);
+ }
+
+ return propertyKey;
+ }
+
+ private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
+ TitanManagement management = titanGraph.getManagementSystem();
+ PropertyKey propertyKey = management.getPropertyKey(propertyName);
+ if (propertyKey == null) {
+ // ignored cardinality as Can only index single-valued property keys on vertices
+ propertyKey = management
+ .makePropertyKey(propertyName)
+ .dataType(propertyClass)
+ .make();
+
+ if (propertyClass == Boolean.class) {
+ //Use standard index as backing index only supports string, int and geo types
+ management.buildIndex(propertyName, Vertex.class).addKey(propertyKey).buildCompositeIndex();
+ management.commit();
+ } else {
+ //Use backing index
+ TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
+ management.addIndexKey(vertexIndex, propertyKey);
+ management.commit();
+ }
+ LOG.info("Created mixed vertex index for property {}", propertyName);
+ }
+
+ return propertyKey;
+ }
+
+ /* Commenting this out since we do not need an index for edge label here
+ private void createEdgeMixedIndex(String propertyName) {
+ TitanManagement management = titanGraph.getManagementSystem();
+ EdgeLabel edgeLabel = management.getEdgeLabel(propertyName);
+ if (edgeLabel == null) {
+ edgeLabel = management.makeEdgeLabel(propertyName).make();
+ management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
+ management.commit();
+ LOG.info("Created index for edge label {}", propertyName);
+ }
+ } */
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
new file mode 100755
index 0000000..02036f4
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanVertex;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.blueprints.GraphQuery;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Utility class for graph operations.
+ */
+public final class GraphHelper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GraphHelper.class);
+
+ private GraphHelper() {
+ }
+
+ public static Vertex createVertexWithIdentity(Graph graph,
+ ITypedReferenceableInstance typedInstance,
+ Set<String> superTypeNames) {
+ final Vertex vertexWithIdentity = createVertexWithoutIdentity(
+ graph, typedInstance.getTypeName(), typedInstance.getId(), superTypeNames);
+
+ // add identity
+ final String guid = UUID.randomUUID().toString();
+ vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid);
+
+ return vertexWithIdentity;
+ }
+
+ public static Vertex createVertexWithoutIdentity(Graph graph,
+ String typeName,
+ Id typedInstanceId,
+ Set<String> superTypeNames) {
+ final Vertex vertexWithoutIdentity = graph.addVertex(null);
+
+ // add type information
+ vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+
+ // add super types
+ for (String superTypeName : superTypeNames) {
+ ((TitanVertex) vertexWithoutIdentity).addProperty(
+ Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
+ }
+
+ // add version information
+ vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
+
+ // add timestamp information
+ vertexWithoutIdentity.setProperty(
+ Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
+
+ return vertexWithoutIdentity;
+ }
+
+ public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,
+ String edgeLabel) {
+ LOG.debug("Adding edge for {} -> label {} -> {}",
+ fromVertex, edgeLabel, toVertex);
+
+ return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
+ }
+
+ public static Vertex findVertexByGUID(TitanGraph titanGraph,
+ String value) {
+ LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value);
+
+ GraphQuery query = titanGraph.query()
+ .has(Constants.GUID_PROPERTY_KEY, value);
+ Iterator<Vertex> results = query.vertices().iterator();
+ // returning one since guid should be unique
+ return results.hasNext() ? results.next() : null;
+ }
+
+ public static String vertexString(final Vertex vertex) {
+ StringBuilder properties = new StringBuilder();
+ for (String propertyKey : vertex.getPropertyKeys()) {
+ properties.append(propertyKey)
+ .append("=").append(vertex.getProperty(propertyKey))
+ .append(", ");
+ }
+
+ return "v[" + vertex.getId() + "], Properties[" + properties + "]";
+ }
+
+ public static String edgeString(final Edge edge) {
+ return "e[" + edge.getLabel() + "], ["
+ + edge.getVertex(Direction.OUT)
+ + " -> " + edge.getLabel() + " -> "
+ + edge.getVertex(Direction.IN)
+ + "]";
+ }
+
+/*
+ public static void dumpToLog(final Graph graph) {
+ LOG.debug("*******************Graph Dump****************************");
+ LOG.debug("Vertices of {}", graph);
+ for (Vertex vertex : graph.getVertices()) {
+ LOG.debug(vertexString(vertex));
+ }
+
+ LOG.debug("Edges of {}", graph);
+ for (Edge edge : graph.getEdges()) {
+ LOG.debug(edgeString(edge));
+ }
+ LOG.debug("*******************Graph Dump****************************");
+ }
+*/
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java
new file mode 100755
index 0000000..5472081
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphProvider.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.inject.throwingproviders.CheckedProvider;
+import com.tinkerpop.blueprints.Graph;
+
+public interface GraphProvider<T extends Graph> extends CheckedProvider<T> {
+
+ @Override
+ T get();
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
new file mode 100755
index 0000000..b26c32b
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import com.google.inject.Provides;
+import com.thinkaurelius.titan.core.TitanFactory;
+import com.thinkaurelius.titan.core.TitanGraph;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.PropertiesUtil;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Singleton;
+import java.util.Iterator;
+
+/**
+ * Default implementation for Graph Provider that doles out Titan Graph.
+ */
+public class TitanGraphProvider implements GraphProvider<TitanGraph> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TitanGraphProvider.class);
+
+ /**
+ * Constant for the configuration property that indicates the prefix.
+ */
+ private static final String ATLAS_PREFIX = "atlas.graph.";
+
+ private static Configuration getConfiguration() throws MetadataException {
+ PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties();
+
+ Configuration graphConfig = new PropertiesConfiguration();
+
+ final Iterator<String> iterator = configProperties.getKeys();
+ while (iterator.hasNext()) {
+ String key = iterator.next();
+ if (key.startsWith(ATLAS_PREFIX)) {
+ String value = (String) configProperties.getProperty(key);
+ key = key.substring(ATLAS_PREFIX.length());
+ graphConfig.setProperty(key, value);
+ LOG.info("Using graph property {}={}", key, value);
+ }
+ }
+
+ return graphConfig;
+ }
+
+ @Override
+ @Singleton
+ @Provides
+ public TitanGraph get() {
+ Configuration config;
+ try {
+ config = getConfiguration();
+ } catch (MetadataException e) {
+ throw new RuntimeException(e);
+ }
+
+ return TitanFactory.open(config);
+ }
+}
[23/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStoreTest.java
deleted file mode 100755
index 1d2684d..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStoreTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.typestore;
-
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import junit.framework.Assert;
-import org.apache.hadoop.metadata.GraphTransaction;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.RepositoryMetadataModule;
-import org.apache.hadoop.metadata.TestUtils;
-import org.apache.hadoop.metadata.repository.graph.GraphHelper;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-import java.util.List;
-
-@Guice(modules = RepositoryMetadataModule.class)
-public class GraphBackedTypeStoreTest {
- @Inject
- private GraphProvider<TitanGraph> graphProvider;
-
- @Inject
- private ITypeStore typeStore;
-
- private TypeSystem ts;
-
- @BeforeClass
- public void setUp() throws Exception {
- ts = TypeSystem.getInstance();
- ts.reset();
- TestUtils.defineDeptEmployeeTypes(ts);
- }
-
- @Test
- @GraphTransaction
- public void testStore() throws MetadataException {
- typeStore.store(ts);
- dumpGraph();
- }
-
- private void dumpGraph() {
- TitanGraph graph = graphProvider.get();
- for (Vertex v : graph.getVertices()) {
- System.out.println("****v = " + GraphHelper.vertexString(v));
- for (Edge e : v.getEdges(Direction.OUT)) {
- System.out.println("****e = " + GraphHelper.edgeString(e));
- }
- }
- }
-
- @Test (dependsOnMethods = "testStore")
- @GraphTransaction
- public void testRestore() throws Exception {
- TypesDef types = typeStore.restore();
-
- //validate enum
- List<EnumTypeDefinition> enumTypes = types.enumTypesAsJavaList();
- Assert.assertEquals(1, enumTypes.size());
- EnumTypeDefinition orgLevel = enumTypes.get(0);
- Assert.assertEquals(orgLevel.name, "OrgLevel");
- Assert.assertEquals(orgLevel.enumValues.length, 2);
- EnumValue enumValue = orgLevel.enumValues[0];
- Assert.assertEquals(enumValue.value, "L1");
- Assert.assertEquals(enumValue.ordinal, 1);
-
- //validate class
- List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
- Assert.assertEquals(1, structTypes.size());
-
- boolean clsTypeFound = false;
- List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
- for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
- if (classType.typeName.equals("Manager")) {
- ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
- Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
- Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
- clsTypeFound = true;
- }
- }
- Assert.assertTrue("Manager type not restored", clsTypeFound);
-
- //validate trait
- List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList();
- Assert.assertEquals(1, traitTypes.size());
- HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
- Assert.assertEquals("SecurityClearance", trait.typeName);
- Assert.assertEquals(1, trait.attributeDefinitions.length);
- AttributeDefinition attribute = trait.attributeDefinitions[0];
- Assert.assertEquals("level", attribute.name);
- Assert.assertEquals(DataTypes.INT_TYPE.getName(), attribute.dataTypeName);
-
- //validate the new types
- ts.reset();
- ts.defineTypes(types);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/ExpressionTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/ExpressionTest.scala b/repository/src/test/scala/org/apache/atlas/query/ExpressionTest.scala
new file mode 100755
index 0000000..3c624ae
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/ExpressionTest.scala
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import org.apache.atlas.query.Expressions._
+import org.apache.atlas.repository.BaseTest
+import org.junit.{Before, Test}
+
+class ExpressionTest extends BaseTest {
+
+ @Before
+ override def setup {
+ super.setup
+
+ QueryTestsUtils.setupTypes
+
+ }
+
+ @Test def testClass: Unit = {
+ val e = QueryProcessor.validate(_class("DB"))
+ println(e)
+ }
+
+ @Test def testFilter: Unit = {
+ val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting"))))
+ println(e)
+ }
+
+ @Test def testSelect: Unit = {
+ val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting"))).
+ select(id("name"), id("owner")))
+ println(e)
+ }
+
+ @Test def testNegTypeTest: Unit = {
+ try {
+ val e = QueryProcessor.validate(_class("DB").where(id("name")))
+ println(e)
+ } catch {
+ case e: ExpressionException if e.getMessage.endsWith("expression: DB where name") => ()
+ }
+ }
+
+ @Test def testIsTrait: Unit = {
+ val e = QueryProcessor.validate(_class("DB").where(isTrait("JdbcAccess")))
+ println(e)
+ }
+
+ @Test def testIsTraitNegative: Unit = {
+ try {
+ val e = QueryProcessor.validate(_class("DB").where(isTrait("Jdb")))
+ println(e)
+ } catch {
+ case e: ExpressionException if e.getMessage.endsWith("not a TraitType, expression: is Jdb") => ()
+ }
+ }
+
+ @Test def testhasField: Unit = {
+ val e = QueryProcessor.validate(_class("DB").where(hasField("name")))
+ println(e)
+ }
+
+ @Test def testHasFieldNegative: Unit = {
+ try {
+ val e = QueryProcessor.validate(_class("DB").where(hasField("nam")))
+ println(e)
+ } catch {
+ case e: ExpressionException if e.getMessage.endsWith("not a TraitType, expression: is Jdb") => ()
+ }
+ }
+
+ @Test def testFieldReference: Unit = {
+ val e = QueryProcessor.validate(_class("DB").field("Table"))
+ println(e)
+ }
+
+ @Test def testNegFieldReference: Unit = {
+ try {
+ val e = QueryProcessor.validate(_class("DB").where(_class("LoadProcess").hasField("name")))
+ println(e)
+ } catch {
+ case e: ExpressionException
+ if e.getMessage.endsWith("srcType of field doesn't match input type, expression: LoadProcess has name") => ()
+ }
+ }
+
+ @Test def testFieldReferenceRedundant: Unit = {
+ val e = QueryProcessor.validate(_class("DB").where(_class("DB").hasField("name")))
+ println(e)
+ }
+
+ @Test def testBackReference: Unit = {
+ val e = QueryProcessor.validate(
+ _class("DB").as("db1").field("Table").where(id("db1").field("name").`=`(string("Reporting"))))
+ println(e)
+ }
+
+ @Test def testArith: Unit = {
+ val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting"))).
+ select(id("name"), id("createTime") + int(1)))
+ println(e)
+ }
+
+ @Test def testComparisonLogical: Unit = {
+ val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting")).
+ and(id("createTime") + int(1) > int(0))))
+ println(e)
+ }
+
+ @Test def testJoinAndSelect1: Unit = {
+ val e = QueryProcessor.validate(
+ _class("DB").as("db1").field("Table").as("tab").where((id("db1").field("createTime") + int(1) > int(0))
+ .and(id("db1").field("name").`=`(string("Reporting")))).select(id("db1").field("name").as("dbName"),
+ id("tab").field("name").as("tabName"))
+ )
+ println(e)
+ }
+
+ @Test def testJoinAndSelect2: Unit = {
+ val e = QueryProcessor.validate(
+ _class("DB").as("db1").field("Table").as("tab").where((id("db1").field("createTime") + int(1) > int(0))
+ .or(id("db1").field("name").`=`(string("Reporting"))))
+ .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName"))
+ )
+ println(e)
+ }
+
+ @Test def testJoinAndSelect3: Unit = {
+ val e = QueryProcessor.validate(
+ _class("DB").as("db1").field("Table").as("tab").where((id("db1").field("createTime") + int(1) > int(0))
+ .and(id("db1").field("name").`=`(string("Reporting")))
+ .or(id("db1").hasField("owner")))
+ .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName"))
+ )
+ println(e)
+ }
+
+ @Test def testJoinAndSelect4: Unit = {
+ val e = QueryProcessor.validate(
+ _class("DB") as "db1" join "Table" as "tab" where (
+ id("db1").field("createTime") + int(1) > int(0) and
+ (id("db1") `.` "name" `=` string("Reporting")) or
+ (id("db1") hasField "owner")
+ ) select(
+ id("db1") `.` "name" as "dbName", id("tab") `.` "name" as "tabName"
+ )
+ )
+ println(e)
+ }
+
+ @Test def testLineageAll: Unit = {
+ val e = QueryProcessor.validate(_class("Table").loop(id("LoadProcess").field("outputTable")))
+ println(e)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
new file mode 100755
index 0000000..aa41313
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import com.thinkaurelius.titan.core.TitanGraph
+import org.apache.atlas.query.Expressions._
+import org.apache.atlas.typesystem.types.TypeSystem
+import org.junit.runner.RunWith
+import org.scalatest.Matchers._
+import org.scalatest._
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
+
+ var g: TitanGraph = null
+
+ override def beforeAll() {
+ TypeSystem.getInstance().reset()
+ QueryTestsUtils.setupTypes
+ g = QueryTestsUtils.setupTestGraph
+ }
+
+ override def afterAll() {
+ g.shutdown()
+ }
+
+ test("testClass") {
+ val r = QueryProcessor.evaluate(_class("DB"), g)
+ validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n
\"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
+ }
+
+ test("testName") {
+ val r = QueryProcessor.evaluate(_class("DB").field("name"), g)
+ validateJson(r, "{\n \"query\":\"DB.name\",\n \"dataType\":\"string\",\n \"rows\":[\n \"Sales\",\n \"Reporting\"\n ]\n}")
+ }
+
+ test("testFilter") {
+ var r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))), g)
+ validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n
\"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
+ }
+
+ test("testFilter2") {
+ var r = QueryProcessor.evaluate(_class("DB").where(id("DB").field("name").`=`(string("Reporting"))), g)
+ validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n
\"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
+ }
+
+
+ test("testSelect") {
+ val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))).
+ select(id("name"), id("owner")), g)
+ validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _col_0, _src1.owner as _col_1\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct1\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_col_0\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"_col_1\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct1\",\n
\"_col_1\":\"Jane BI\",\n \"_col_0\":\"Reporting\"\n }\n ]\n}")
+ }
+
+ test("testIsTrait") {
+ val r = QueryProcessor.evaluate(_class("Table").where(isTrait("Dimension")), g)
+ validateJson(r, "{\n \"query\":\"Table where Table is Dimension\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\
":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n
\"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n }\n ]\n}")
+ }
+
+ test("testhasField") {
+ val r = QueryProcessor.evaluate(_class("DB").where(hasField("name")), g)
+ validateJson(r, "{\n \"query\":\"DB where DB has name\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeNam
e\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
+ }
+
+ test("testFieldReference") {
+ val r = QueryProcessor.evaluate(_class("DB").field("Table"), g)
+ validateJson(r, "{\n \"query\":\"DB Table\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\":\"StorageDesc\",\n
\"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"2048\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"512\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"2
56\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n
},\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"8960\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"7424\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"12800\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"11264\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\"
,\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
+ }
+
+ test("testBackReference") {
+ val r = QueryProcessor.evaluate(
+ _class("DB").as("db").field("Table").where(id("db").field("name").`=`(string("Reporting"))), g)
+ validateJson(r, null)
+ }
+
+ test("testArith") {
+ val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))).
+ select(id("name"), id("createTime") + int(1)), g)
+ validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _col_0, (_src1.createTime + 1) as _col_1\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct3\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_col_0\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"_col_1\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStru
ct3\",\n \"_col_1\":1501,\n \"_col_0\":\"Reporting\"\n }\n ]\n}")
+ }
+
+ test("testComparisonLogical") {
+ val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting")).
+ and(id("createTime") > int(0))), g)
+ validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.atlas.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"
createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
+ }
+
+ test("testJoinAndSelect1") {
+ val r = QueryProcessor.evaluate(
+ _class("DB").as("db1").where(id("name").`=`(string("Sales"))).field("Table").as("tab").
+ where((isTrait("Dimension"))).
+ select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g
+ )
+ validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct5\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n
]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
+ }
+
+ test("testJoinAndSelect2") {
+ val r = QueryProcessor.evaluate(
+ _class("DB").as("db1").where((id("db1").field("createTime") > int(0))
+ .or(id("name").`=`(string("Reporting")))).field("Table").as("tab")
+ .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g
+ )
+ validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\"
:\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
+ }
+
+ test("testJoinAndSelect3") {
+ val r = QueryProcessor.evaluate(
+ _class("DB").as("db1").where((id("db1").field("createTime") > int(0))
+ .and(id("db1").field("name").`=`(string("Reporting")))
+ .or(id("db1").hasField("owner"))).field("Table").as("tab")
+ .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g
+ )
+ validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or DB as db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\
n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
+ }
+
+ test("testJoinAndSelect4") {
+ val r = QueryProcessor.evaluate(
+ _class("DB").as("db1").where(id("name").`=`(string("Sales"))).field("Table").as("tab").
+ where((isTrait("Dimension"))).
+ select(id("db1").as("dbO"), id("tab").field("name").as("tabName")), g
+ )
+ validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1 as dbO, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbO\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$
typeName$\":\"\",\n \"dbO\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
+ }
+
+ test("testNegativeInvalidType") {
+ val p = new QueryParser
+ val e = p("from blah").right.get
+ an [ExpressionException] should be thrownBy QueryProcessor.evaluate(e, g)
+ }
+
+ test("Bug37860") {
+ val p = new QueryParser
+ val e = p("Table as t where name = 'sales_fact' db where name = 'Sales' and owner = 'John ETL' select t").right.get
+ val r = QueryProcessor.evaluate(e, g)
+ validateJson(r)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala b/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala
new file mode 100755
index 0000000..ea0b9bb
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import com.thinkaurelius.titan.core.TitanGraph
+import org.apache.atlas.query.Expressions._
+import org.apache.atlas.typesystem.types.TypeSystem
+import org.junit.runner.RunWith
+import org.scalatest._
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class GremlinTest2 extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
+
+ var g: TitanGraph = null
+
+ override def beforeAll() {
+ TypeSystem.getInstance().reset()
+ QueryTestsUtils.setupTypes
+ g = QueryTestsUtils.setupTestGraph
+ }
+
+ override def afterAll() {
+ g.shutdown()
+ }
+
+ test("testTraitSelect") {
+ val r = QueryProcessor.evaluate(_class("Table").as("t").join("Dimension").as("dim").select(id("t"), id("dim")), g)
+ validateJson(r, "{\n \"query\":\"Table as t.Dimension as dim select t as _col_0, dim as _col_1\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_col_0\",\n \"dataTypeName\":\"Table\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"_col_1\",\n \"dataTypeName\":\"Dimension\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"\",\n \"_col_1\":{\n \"$typeName$\":\"Dimension\"\n },\n \"_col_0\"
:{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n }\n },\n {\n \"$typeName$\":\"\",\n \"_col_1\":{\n \"$typeName$\":\"Dimension\"\n },\n \"_col_0\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n }\n },\n {\n \"$typeName$\":\"\",\n \"_col_1\":{\n \"$typeName$\":\"Dimension\"\n },\n \"_col_0\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n }\n }\n ]\n}")
+ }
+
+ test("testTrait") {
+ val r = QueryProcessor.evaluate(_trait("Dimension"), g)
+ validateJson(r)
+ }
+
+ test("testTraitInstance") {
+ val r = QueryProcessor.evaluate(_trait("Dimension").traitInstance(), g)
+ validateJson(r)
+ }
+
+ test("testInstanceAddedToFilter") {
+ val r = QueryProcessor.evaluate(_trait("Dimension").hasField("typeName"), g)
+ validateJson(r)
+ }
+
+ test("testInstanceFilter") {
+ val r = QueryProcessor.evaluate(_trait("Dimension").traitInstance().hasField("name"), g)
+ validateJson(r)
+ }
+
+ test("testLineageWithPath") {
+ val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")).path(), g)
+ validateJson(r)
+ }
+
+ test("testLineageAllSelectWithPath") {
+ val r = QueryProcessor.evaluate(_class("Table").as("src").loop(id("LoadProcess").field("outputTable")).as("dest").
+ select(id("src").field("name").as("srcTable"), id("dest").field("name").as("destTable")).path(), g)
+ validateJson(r)
+ }
+
+ test("testLineageAllSelectWithPathFromParser") {
+ val p = new QueryParser
+ val e = p("Table as src loop (LoadProcess outputTable) as dest " +
+ "select src.name as srcTable, dest.name as destTable withPath").right.get
+ //Table as src loop (LoadProcess where LoadProcess.outputTable) as dest select src.name as srcTable, dest.name as destTable withPath
+ val r = QueryProcessor.evaluate(e, g)
+ validateJson(r)
+ }
+
+ test("testLineageAllSelectWithPathFromParser2") {
+ val p = new QueryParser
+
+ val e = p("Table as src loop (`LoadProcess->outputTable` inputTables) as dest " +
+ "select src.name as srcTable, dest.name as destTable withPath").right.get
+ val r = QueryProcessor.evaluate(e, g)
+ validateJson(r)
+ }
+
+ test("testHighLevelLineage") {
+ val r = HiveLineageQuery("Table", "sales_fact_monthly_mv",
+ "LoadProcess",
+ "inputTables",
+ "outputTable",
+ None, Some(List("name")), true, GraphPersistenceStrategy1, g).evaluate()
+ validateJson(r)
+ }
+
+ test("testHighLevelLineageReturnGraph") {
+ val r = HiveLineageQuery("Table", "sales_fact_monthly_mv",
+ "LoadProcess",
+ "inputTables",
+ "outputTable",
+ None, Some(List("name")), true, GraphPersistenceStrategy1, g).graph
+
+ println(r.toInstanceJson)
+ //validateJson(r)
+ }
+
+ test("testHighLevelWhereUsed") {
+ val r = HiveWhereUsedQuery("Table", "sales_fact",
+ "LoadProcess",
+ "inputTables",
+ "outputTable",
+ None, Some(List("name")), true, GraphPersistenceStrategy1, g).evaluate()
+ validateJson(r)
+ }
+
+ test("testHighLevelWhereUsedReturnGraph") {
+ val r = HiveWhereUsedQuery("Table", "sales_fact",
+ "LoadProcess",
+ "inputTables",
+ "outputTable",
+ None, Some(List("name")), true, GraphPersistenceStrategy1, g).graph
+ println(r.toInstanceJson)
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala b/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala
new file mode 100755
index 0000000..71cbad3
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import java.io.File
+import java.util.UUID
+import java.util.concurrent.atomic.AtomicInteger
+import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
+
+import com.thinkaurelius.titan.core.TitanGraph
+import com.typesafe.config.ConfigFactory
+import org.apache.commons.io.FileUtils
+
+import scala.collection.mutable.ArrayBuffer
+
+object HiveTitanSample {
+
+ private var nextVertexId: AtomicInteger = new AtomicInteger(0)
+ private var nextEdgeId: AtomicInteger = new AtomicInteger(1000)
+
+ trait Vertex {
+ val _id: String
+
+ def id = _id
+ val version = 0
+ val guid = s"""${UUID.randomUUID()}""".stripMargin
+
+ def addEdge(to: Vertex, label: String, edges: ArrayBuffer[String]): Unit = {
+ edges +=
+ s"""{"_id" : "${nextEdgeId.incrementAndGet()}", "_type" : "edge", "_inV" : "${to.id}", "_outV" : "$id", "_label" : "$label"}"""
+ }
+
+ def toGSon(vertices: ArrayBuffer[String],
+ edges: ArrayBuffer[String]): Unit = {
+
+ val sb = new StringBuilder
+ sb.append( s"""{"typeName" : "${this.getClass.getSimpleName}", "_type" : "vertex"""")
+
+ this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f =>
+ f.setAccessible(true)
+ var fV = f.get(this)
+ fV = fV match {
+ case _: String => s""""$fV""""
+ case _ => fV
+ }
+
+ fV match {
+ case x: Vertex => addEdge(x, s"${this.getClass.getSimpleName}.${f.getName}", edges)
+ case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex],
+ s"${this.getClass.getSimpleName}.${f.getName}", edges))
+ case _ => sb.append( s""", "${f.getName}" : $fV""")
+ sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $fV""")
+ }
+ }
+
+ this.getClass.getDeclaredFields filter (_.getName == "traits") foreach { f =>
+ f.setAccessible(true)
+ var traits = f.get(this).asInstanceOf[Option[List[Trait]]]
+
+ if (traits.isDefined) {
+ val fV = traits.get.map(_.getClass.getSimpleName).mkString(",")
+ sb.append( s""", "traitNames" : "$fV"""")
+ }
+ }
+
+ sb.append("}")
+ vertices += sb.toString()
+ }
+ }
+
+ trait Trait extends Vertex
+
+ trait Struct extends Vertex
+
+ trait Instance extends Vertex {
+ val traits: Option[List[Trait]]
+
+ override def toGSon(vertices: ArrayBuffer[String],
+ edges: ArrayBuffer[String]): Unit = {
+ super.toGSon(vertices, edges)
+
+ if (traits.isDefined) {
+ traits.get foreach { t =>
+ t.toGSon(vertices, edges)
+ addEdge(t, s"${this.getClass.getSimpleName}.${t.getClass.getSimpleName}", edges)
+ }
+ }
+ }
+
+ }
+
+ case class JdbcAccess(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
+
+ case class PII(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
+
+ case class Dimension(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
+
+ case class Metric(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
+
+ case class ETL(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
+
+
+ case class DB(name: String, owner: String, createTime: Int, traits: Option[List[Trait]] = None,
+ _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
+
+ case class StorageDescriptor(inputFormat: String, outputFormat: String,
+ _id: String = "" + nextVertexId.incrementAndGet()) extends Struct
+
+ case class Column(name: String, dataType: String, sd: StorageDescriptor,
+ traits: Option[List[Trait]] = None,
+ _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
+
+ case class Table(name: String, db: DB, sd: StorageDescriptor,
+ traits: Option[List[Trait]] = None,
+ _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
+
+ case class TableDef(name: String, db: DB, inputFormat: String, outputFormat: String,
+ columns: List[(String, String, Option[List[Trait]])],
+ traits: Option[List[Trait]] = None) {
+ val sd = StorageDescriptor(inputFormat, outputFormat)
+ val colDefs = columns map { c =>
+ Column(c._1, c._2, sd, c._3)
+ }
+ val tablDef = Table(name, db, sd, traits)
+
+ def toGSon(vertices: ArrayBuffer[String],
+ edges: ArrayBuffer[String]): Unit = {
+ sd.toGSon(vertices, edges)
+ colDefs foreach {
+ _.toGSon(vertices, edges)
+ }
+ tablDef.toGSon(vertices, edges)
+ }
+ }
+
+ case class LoadProcess(name: String, inputTables: List[Vertex],
+ outputTable: Vertex,
+ traits: Option[List[Trait]] = None,
+ _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
+
+ case class View(name: String, db: DB, inputTables: List[Vertex],
+ traits: Option[List[Trait]] = None,
+ _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
+
+ val salesDB = DB("Sales", "John ETL", 1000)
+ val salesFact = TableDef("sales_fact",
+ salesDB,
+ "TextInputFormat",
+ "TextOutputFormat",
+ List(
+ ("time_id", "int", None),
+ ("product_id", "int", None),
+ ("customer_id", "int", None),
+ ("sales", "double", Some(List(Metric())))
+ ))
+ val productDim = TableDef("product_dim",
+ salesDB,
+ "TextInputFormat",
+ "TextOutputFormat",
+ List(
+ ("product_id", "int", None),
+ ("product_name", "string", None),
+ ("brand_name", "string", None)
+ ),
+ Some(List(Dimension())))
+ val timeDim = TableDef("time_dim",
+ salesDB,
+ "TextInputFormat",
+ "TextOutputFormat",
+ List(
+ ("time_id", "int", None),
+ ("dayOfYear", "int", None),
+ ("weekDay", "string", None)
+ ),
+ Some(List(Dimension())))
+ val customerDim = TableDef("customer_dim",
+ salesDB,
+ "TextInputFormat",
+ "TextOutputFormat",
+ List(
+ ("customer_id", "int", None),
+ ("name", "int", None),
+ ("address", "string", Some(List(PII())))
+ ),
+ Some(List(Dimension())))
+
+ val reportingDB = DB("Reporting", "Jane BI", 1500)
+ val salesFactDaily = TableDef("sales_fact_daily_mv",
+ reportingDB,
+ "TextInputFormat",
+ "TextOutputFormat",
+ List(
+ ("time_id", "int", None),
+ ("product_id", "int", None),
+ ("customer_id", "int", None),
+ ("sales", "double", Some(List(Metric())))
+ ))
+ val loadSalesFactDaily = LoadProcess("loadSalesDaily",
+ List(salesFact.tablDef, timeDim.tablDef), salesFactDaily.tablDef,
+ Some(List(ETL())))
+
+
+ val productDimView = View("product_dim_view", reportingDB,
+ List(productDim.tablDef),
+ Some(List(Dimension(), JdbcAccess())))
+
+ val customerDimView = View("customer_dim_view", reportingDB,
+ List(customerDim.tablDef),
+ Some(List(Dimension(), JdbcAccess())))
+
+ val salesFactMonthly = TableDef("sales_fact_monthly_mv",
+ reportingDB,
+ "TextInputFormat",
+ "TextOutputFormat",
+ List(
+ ("time_id", "int", None),
+ ("product_id", "int", None),
+ ("customer_id", "int", None),
+ ("sales", "double", Some(List(Metric())))
+ ))
+ val loadSalesFactMonthly = LoadProcess("loadSalesMonthly",
+ List(salesFactDaily.tablDef), salesFactMonthly.tablDef,
+ Some(List(ETL())))
+
+
+ val vertices: ArrayBuffer[String] = new ArrayBuffer[String]()
+ val edges: ArrayBuffer[String] = new ArrayBuffer[String]()
+
+ salesDB.toGSon(vertices, edges)
+ salesFact.toGSon(vertices, edges)
+ productDim.toGSon(vertices, edges)
+ timeDim.toGSon(vertices, edges)
+ customerDim.toGSon(vertices, edges)
+
+ reportingDB.toGSon(vertices, edges)
+ salesFactDaily.toGSon(vertices, edges)
+ loadSalesFactDaily.toGSon(vertices, edges)
+ productDimView.toGSon(vertices, edges)
+ customerDimView.toGSon(vertices, edges)
+ salesFactMonthly.toGSon(vertices, edges)
+ loadSalesFactMonthly.toGSon(vertices, edges)
+
+ def toGSon(): String = {
+ s"""{
+ "mode":"NORMAL",
+ "vertices": ${vertices.mkString("[\n\t", ",\n\t", "\n]")},
+ "edges": ${edges.mkString("[\n\t", ",\n\t", "\n]")}
+ }
+ """.stripMargin
+ }
+
+ def writeGson(fileName: String): Unit = {
+ FileUtils.writeStringToFile(new File(fileName), toGSon())
+ }
+
+ val GremlinQueries = List(
+ // 1. List all DBs
+ """g.V.has("typeName", "DB")""",
+
+ // 2. List all DB nmes
+ """g.V.has("typeName", "DB").name""",
+
+ // 3. List all Tables in Reporting DB
+ """g.V.has("typeName", "DB").has("name", "Reporting").inE("Table.db").outV""",
+ """g.V.has("typeName", "DB").as("db").inE("Table.db").outV.and(_().back("db").has("name", "Reporting"))""",
+
+ // 4. List all Tables in Reporting DB, list as D.name, Tbl.name
+ """
+ g.V.has("typeName", "DB").has("name", "Reporting").as("db").inE("Table.db").outV.as("tbl").select{it.name}{it.name}
+ """.stripMargin,
+
+ // 5. List all tables that are Dimensions and have the TextInputFormat
+ """
+ g.V.as("v").and(_().outE("Table.Dimension"), _().out("Table.sd").has("inputFormat", "TextInputFormat")).name
+ """.stripMargin,
+
+ // 6. List all tables that are Dimensions or have the TextInputFormat
+ """
+ g.V.as("v").or(_().outE("Table.Dimension"), _().out("Table.sd").has("inputFormat", "TextInputFormat")).name
+ """.stripMargin,
+
+ // 7. List tables that have at least 1 PII column
+ """
+ g.V.has("typeName", "Table").as("tab").out("Table.sd").in("Column.sd").as("column"). \
+ out("Column.PII").select.groupBy{it.getColumn("tab")}{it.getColumn("column")}{[ "c" : it.size]}.cap.scatter.filter{it.value.c > 0}. \
+ transform{it.key}.name """.stripMargin
+
+ // 7.a from Table as tab -> g.V.has("typeName", "Table").as("tab")
+ // 7.b sd.Column as column -> out("Table.sd").in("Column.sd").as("column")
+ // 7.c is PII -> out("Column.PII")
+ // 7.d select tab, column -> select{it}{it}
+ // 7.e groupBy tab compute count(column) as c
+ // 7.f where c > 0
+
+ // 7.a Alias(Type("Table"), "tab")
+ // 7b. Field("sd", Alias(Type("Table"), "tab"))
+ // Alias(Field("Column", Field("sd", Alias(Type("Table"), "tab"))), "column")
+ // 7.c Filter(is("PII"), Alias(Field("Column", Field("sd", Alias(Type("Table"), "tab"))), "column"))
+ // 7.d
+ )
+}
+
+object TestApp extends App with GraphUtils {
+
+ var conf = ConfigFactory.load()
+ conf = conf.getConfig("graphRepo")
+ val g: TitanGraph = titanGraph(conf)
+ val manager: ScriptEngineManager = new ScriptEngineManager
+ val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
+ val bindings: Bindings = engine.createBindings
+ bindings.put("g", g)
+
+ val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
+ HiveTitanSample.writeGson(hiveGraphFile)
+ bindings.put("hiveGraphFile", hiveGraphFile)
+
+ try {
+ engine.eval("g.loadGraphSON(hiveGraphFile)", bindings)
+
+ println(engine.eval("g.V.typeName.toList()", bindings))
+
+ HiveTitanSample.GremlinQueries.foreach { q =>
+ println(q)
+ println("Result: " + engine.eval(q + ".toList()", bindings))
+ }
+ } finally {
+ g.shutdown()
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/LexerTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/LexerTest.scala b/repository/src/test/scala/org/apache/atlas/query/LexerTest.scala
new file mode 100755
index 0000000..5abd797
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/LexerTest.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import org.junit.{Assert, Test}
+
+import scala.util.parsing.input.CharArrayReader
+
+class LexerTest {
+
+ def scan(p: QueryParser, str: String): p.lexical.ParseResult[_] = {
+ val l = p.lexical
+ var s: l.Input = new CharArrayReader(str.toCharArray)
+ var r = (l.whitespace.? ~ l.token)(s)
+ s = r.next
+
+ while (r.successful && !s.atEnd) {
+ s = r.next
+ if (!s.atEnd) {
+ r = (l.whitespace.? ~ l.token)(s)
+ }
+ }
+ r.asInstanceOf[p.lexical.ParseResult[_]]
+ }
+
+ @Test def testSimple {
+ val p = new QueryParser
+ val r = scan(p, """DB where db1.name""")
+ Assert.assertTrue(r.successful)
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
new file mode 100755
index 0000000..58f3d98
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
@@ -0,0 +1,515 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import com.thinkaurelius.titan.core.TitanGraph
+import org.apache.atlas.query.Expressions._
+import org.apache.atlas.typesystem.types.TypeSystem
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite}
+
+@RunWith(classOf[JUnitRunner])
+class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
+
+ var g: TitanGraph = null
+
+ override def beforeAll() {
+ TypeSystem.getInstance().reset()
+ QueryTestsUtils.setupTypes
+ g = QueryTestsUtils.setupTestGraph
+ }
+
+ override def afterAll() {
+ g.shutdown()
+ }
+
+ val STRUCT_NAME_REGEX = (TypeUtils.TEMP_STRUCT_NAME_PREFIX + "\\d+").r
+ val PREFIX_SPACES_REGEX = ("\\n\\s*").r
+
+ def validateJson(r: GremlinQueryResult, expected: String = null): Unit = {
+ val rJ = r.toJson
+ if (expected != null) {
+ var a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
+ a = PREFIX_SPACES_REGEX.replaceAllIn(a, "")
+ var b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
+ b = PREFIX_SPACES_REGEX.replaceAllIn(b, "")
+ Assertions.assert(a == b)
+ } else {
+ println(rJ)
+ }
+ }
+
+ test("testInputTables") {
+ val r = QueryProcessor.evaluate(_class("LoadProcess").field("inputTables"), g)
+ val x = r.toJson
+ validateJson(r, """{
+ "query":"LoadProcess inputTables",
+ "dataType":{
+ "superTypes":[
+
+ ],
+ "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
+ "typeName":"Table",
+ "attributeDefinitions":[
+ {
+ "name":"name",
+ "dataTypeName":"string",
+ "multiplicity":{
+ "lower":0,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ },
+ {
+ "name":"db",
+ "dataTypeName":"DB",
+ "multiplicity":{
+ "lower":1,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ },
+ {
+ "name":"sd",
+ "dataTypeName":"StorageDesc",
+ "multiplicity":{
+ "lower":1,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ }
+ ]
+ },
+ "rows":[
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"2048",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"512",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"256",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact"
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"4864",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"3840",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"256",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"time_dim",
+ "$traits$":{
+ "Dimension":{
+ "$typeName$":"Dimension"
+ }
+ }
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"8960",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"7424",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_daily_mv"
+ }
+ ]
+}""")
+ }
+
+ test("testLoadProcessOut") {
+ val r = QueryProcessor.evaluate(_class("Table").field("LoadProcess").field("outputTable"), g)
+ validateJson(r, null)
+ }
+
+ test("testLineageAll") {
+ val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")), g)
+ validateJson(r, """{
+ "query":"Table as _loop0 loop (LoadProcess outputTable)",
+ "dataType":{
+ "superTypes":[
+
+ ],
+ "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
+ "typeName":"Table",
+ "attributeDefinitions":[
+ {
+ "name":"name",
+ "dataTypeName":"string",
+ "multiplicity":{
+ "lower":0,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ },
+ {
+ "name":"db",
+ "dataTypeName":"DB",
+ "multiplicity":{
+ "lower":1,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ },
+ {
+ "name":"sd",
+ "dataTypeName":"StorageDesc",
+ "multiplicity":{
+ "lower":1,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ }
+ ]
+ },
+ "rows":[
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"8960",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"7424",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_daily_mv"
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"12800",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"11264",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_monthly_mv"
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"8960",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"7424",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_daily_mv"
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"12800",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"11264",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_monthly_mv"
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"12800",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"11264",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_monthly_mv"
+ }
+ ]
+}""")
+ }
+
+ test("testLineageAllSelect") {
+ val r = QueryProcessor.evaluate(_class("Table").as("src").loop(id("LoadProcess").field("outputTable")).as("dest").
+ select(id("src").field("name").as("srcTable"), id("dest").field("name").as("destTable")), g)
+ validateJson(r, """{
+ "query":"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable",
+ "dataType":{
+ "typeName":"__tempQueryResultStruct2",
+ "attributeDefinitions":[
+ {
+ "name":"srcTable",
+ "dataTypeName":"string",
+ "multiplicity":{
+ "lower":0,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ },
+ {
+ "name":"destTable",
+ "dataTypeName":"string",
+ "multiplicity":{
+ "lower":0,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ }
+ ]
+ },
+ "rows":[
+ {
+ "$typeName$":"__tempQueryResultStruct2",
+ "srcTable":"sales_fact",
+ "destTable":"sales_fact_daily_mv"
+ },
+ {
+ "$typeName$":"__tempQueryResultStruct2",
+ "srcTable":"sales_fact",
+ "destTable":"sales_fact_monthly_mv"
+ },
+ {
+ "$typeName$":"__tempQueryResultStruct2",
+ "srcTable":"time_dim",
+ "destTable":"sales_fact_daily_mv"
+ },
+ {
+ "$typeName$":"__tempQueryResultStruct2",
+ "srcTable":"time_dim",
+ "destTable":"sales_fact_monthly_mv"
+ },
+ {
+ "$typeName$":"__tempQueryResultStruct2",
+ "srcTable":"sales_fact_daily_mv",
+ "destTable":"sales_fact_monthly_mv"
+ }
+ ]
+}""")
+ }
+
+ test("testLineageFixedDepth") {
+ val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable"), int(1)), g)
+ validateJson(r, """{
+ "query":"Table as _loop0 loop (LoadProcess outputTable) times 1",
+ "dataType":{
+ "superTypes":[
+
+ ],
+ "hierarchicalMetaTypeName":"org.apache.atlas.typesystem.types.ClassType",
+ "typeName":"Table",
+ "attributeDefinitions":[
+ {
+ "name":"name",
+ "dataTypeName":"string",
+ "multiplicity":{
+ "lower":0,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ },
+ {
+ "name":"db",
+ "dataTypeName":"DB",
+ "multiplicity":{
+ "lower":1,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ },
+ {
+ "name":"sd",
+ "dataTypeName":"StorageDesc",
+ "multiplicity":{
+ "lower":1,
+ "upper":1,
+ "isUnique":false
+ },
+ "isComposite":false,
+ "isUnique":false,
+ "isIndexable":true,
+ "reverseAttributeName":null
+ }
+ ]
+ },
+ "rows":[
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"8960",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"7424",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_daily_mv"
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"8960",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"7424",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_daily_mv"
+ },
+ {
+ "$typeName$":"Table",
+ "$id$":{
+ "id":"12800",
+ "$typeName$":"Table",
+ "version":0
+ },
+ "sd":{
+ "id":"11264",
+ "$typeName$":"StorageDesc",
+ "version":0
+ },
+ "db":{
+ "id":"7168",
+ "$typeName$":"DB",
+ "version":0
+ },
+ "name":"sales_fact_monthly_mv"
+ }
+ ]
+}""")
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala b/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala
new file mode 100755
index 0000000..602b2b5
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import org.apache.atlas.repository.BaseTest
+import org.junit.{Before, Test}
+
+
+class ParserTest extends BaseTest {
+
+ @Before
+ override def setup {
+ super.setup
+
+ QueryTestsUtils.setupTypes
+
+ }
+
+ @Test def testFrom: Unit = {
+ val p = new QueryParser
+ println(p("from DB").right.get.toString)
+ }
+
+ @Test def testFrom2: Unit = {
+ val p = new QueryParser
+ println(p("DB").right.get.toString)
+ }
+
+ @Test def testJoin1: Unit = {
+ val p = new QueryParser
+ println(p("DB, Table").right.get.toString)
+ }
+
+ @Test def testWhere1: Unit = {
+ val p = new QueryParser
+ println(p("DB as db1 Table where db1.name ").right.get.toString)
+ }
+
+ @Test def testWhere2: Unit = {
+ val p = new QueryParser
+ println(p("DB name = \"Reporting\"").right.get.toString)
+ }
+
+ @Test def testIsTrait: Unit = {
+ val p = new QueryParser
+ println(p("Table isa Dimension").right.get.toString)
+ println(p("Table is Dimension").right.get.toString)
+ }
+
+ @Test def test4: Unit = {
+ val p = new QueryParser
+ println(p("DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1").right.get.toString)
+ }
+
+ @Test def testJoin2: Unit = {
+ val p = new QueryParser
+ println(p("DB as db1 where (createTime + 1) > 0 and (db1.name = \"Reporting\") or DB has owner Table as tab " +
+ " select db1.name as dbName, tab.name as tabName").right.get.toString)
+ }
+
+ @Test def testLoop: Unit = {
+ val p = new QueryParser
+ println(p("Table loop (LoadProcess outputTable)").right.get.toString)
+ }
+
+ @Test def testNegInvalidateType: Unit = {
+ val p = new QueryParser
+ val x = p("from blah")
+ println(p("from blah").left)
+ }
+
+ @Test def testPath1: Unit = {
+ val p = new QueryParser
+ println(p("Table loop (LoadProcess outputTable) withPath").right.get.toString)
+ }
+
+ @Test def testPath2: Unit = {
+ val p = new QueryParser
+ println(p(
+ "Table as src loop (LoadProcess outputTable) as dest " +
+ "select src.name as srcTable, dest.name as destTable withPath").right.get.toString
+ )
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
new file mode 100755
index 0000000..d79758a
--- /dev/null
+++ b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import java.io.File
+import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
+
+import com.google.common.collect.ImmutableList
+import com.thinkaurelius.titan.core.{TitanFactory, TitanGraph}
+import com.tinkerpop.blueprints.Vertex
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.atlas.typesystem.types._
+import org.apache.commons.configuration.{Configuration, ConfigurationException, MapConfiguration}
+import org.apache.commons.io.FileUtils
+import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite}
+
+trait GraphUtils {
+
+ import scala.collection.JavaConversions._
+
+ def getConfiguration(config: Config): Configuration = {
+ val keys = config.entrySet().map {
+ _.getKey
+ }
+ val gConfig: java.util.Map[String, String] = new java.util.HashMap[String, String]()
+ keys.foreach { k =>
+ gConfig.put(k, config.getString(k))
+ }
+ return new MapConfiguration(gConfig)
+ }
+
+
+ def titanGraph(conf: Config) = {
+ try {
+ val g = TitanFactory.open(getConfiguration(conf))
+ val mgmt = g.getManagementSystem
+ val typname = mgmt.makePropertyKey("typeName").dataType(classOf[String]).make()
+ mgmt.buildIndex("byTypeName", classOf[Vertex]).addKey(typname).buildCompositeIndex()
+ mgmt.commit()
+ g
+ } catch {
+ case e: ConfigurationException => throw new RuntimeException(e)
+ }
+ }
+}
+
+object QueryTestsUtils extends GraphUtils {
+
+ def setupTypes: Unit = {
+ def attrDef(name: String, dT: IDataType[_],
+ m: Multiplicity = Multiplicity.OPTIONAL,
+ isComposite: Boolean = false,
+ reverseAttributeName: String = null) = {
+ require(name != null)
+ require(dT != null)
+ new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName)
+ }
+
+ def dbClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "DB", null,
+ Array(
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.INT_TYPE)
+ ))
+
+ def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDesc", null,
+ Array(
+ attrDef("inputFormat", DataTypes.STRING_TYPE),
+ attrDef("outputFormat", DataTypes.STRING_TYPE)
+ ))
+
+ def columnClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Column", null,
+ Array(
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("dataType", DataTypes.STRING_TYPE),
+ new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null)
+ ))
+
+ def tblClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Table", null,
+ Array(
+ attrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null)
+ ))
+
+ def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null,
+ Array(
+ attrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null),
+ new AttributeDefinition("outputTable", "Table", Multiplicity.REQUIRED, false, null)
+ ))
+
+ def viewClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "View", null,
+ Array(
+ attrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null)
+ ))
+
+ def dimTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Dimension", null,
+ Array[AttributeDefinition]())
+ def piiTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "PII", null,
+ Array[AttributeDefinition]())
+ def metricTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Metric", null,
+ Array[AttributeDefinition]())
+ def etlTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "ETL", null,
+ Array[AttributeDefinition]())
+ def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null,
+ Array[AttributeDefinition]())
+
+ TypeSystem.getInstance().defineTypes(ImmutableList.of[StructTypeDefinition],
+ ImmutableList.of[HierarchicalTypeDefinition[TraitType]](dimTraitDef, piiTraitDef,
+ metricTraitDef, etlTraitDef, jdbcTraitDef),
+ ImmutableList.of[HierarchicalTypeDefinition[ClassType]](dbClsDef, storageDescClsDef, columnClsDef, tblClsDef,
+ loadProcessClsDef, viewClsDef))
+
+ ()
+ }
+
+ def setupTestGraph: TitanGraph = {
+ var conf = ConfigFactory.load()
+ conf = conf.getConfig("graphRepo")
+ val g = titanGraph(conf)
+ val manager: ScriptEngineManager = new ScriptEngineManager
+ val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
+ val bindings: Bindings = engine.createBindings
+ bindings.put("g", g)
+
+ val hiveGraphFile = FileUtils.getTempDirectory().getPath.toString + File.separator + System.nanoTime() + ".gson"
+ HiveTitanSample.writeGson(hiveGraphFile)
+ bindings.put("hiveGraphFile", hiveGraphFile)
+
+ engine.eval("g.loadGraphSON(hiveGraphFile)", bindings)
+ g
+ }
+
+}
+
+trait BaseGremlinTest {
+ self : FunSuite with BeforeAndAfterAll =>
+
+ val STRUCT_NAME_REGEX = (TypeUtils.TEMP_STRUCT_NAME_PREFIX + "\\d+").r
+ def validateJson(r: GremlinQueryResult, expected: String = null): Unit = {
+ val rJ = r.toJson
+ if (expected != null) {
+ val a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
+ val b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
+ Assertions.assert(a == b)
+ } else {
+ println(rJ)
+ }
+ }
+
+}
[05/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationFilter.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationFilter.java
deleted file mode 100644
index bac27e2..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationFilter.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.filters;
-
-import com.google.inject.Singleton;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.metadata.security.SecurityProperties;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
-import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.*;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Enumeration;
-import java.util.Iterator;
-import java.util.Properties;
-
-/**
- * This enforces authentication as part of the filter before processing the request.
- * todo: Subclass of {@link org.apache.hadoop.security.authentication.server.AuthenticationFilter}.
- */
-@Singleton
-public class MetadataAuthenticationFilter extends AuthenticationFilter {
- private static final Logger LOG = LoggerFactory.getLogger(MetadataAuthenticationFilter.class);
- static final String PREFIX = "metadata.http.authentication.";
-
- @Override
- protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException {
- PropertiesConfiguration configuration;
- try {
- configuration = PropertiesUtil.getApplicationProperties();
- } catch (Exception e) {
- throw new ServletException(e);
- }
-
- Properties config = new Properties();
-
- config.put(AuthenticationFilter.COOKIE_PATH, "/");
-
- // add any config passed in as init parameters
- Enumeration<String> enumeration = filterConfig.getInitParameterNames();
- while (enumeration.hasMoreElements()) {
- String name = enumeration.nextElement();
- config.put(name, filterConfig.getInitParameter(name));
- }
- // transfer application.properties config items starting with defined prefix
- Iterator<String> itor = configuration.getKeys();
- while (itor.hasNext()) {
- String name = itor.next();
- if (name.startsWith(PREFIX)) {
- String value = configuration.getString(name);
- name = name.substring(PREFIX.length());
- config.put(name, value);
- }
- }
-
- //Resolve _HOST into bind address
- String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS);
- if (bindAddress == null) {
- LOG.info("No host name configured. Defaulting to local host name.");
- try {
- bindAddress = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- throw new ServletException("Unable to obtain host name", e);
- }
- }
- String principal = config.getProperty(KerberosAuthenticationHandler.PRINCIPAL);
- if (principal != null) {
- try {
- principal = SecurityUtil.getServerPrincipal(principal, bindAddress);
- } catch (IOException ex) {
- throw new RuntimeException("Could not resolve Kerberos principal name: " + ex.toString(), ex);
- }
- config.put(KerberosAuthenticationHandler.PRINCIPAL, principal);
- }
-
- LOG.info("AuthenticationFilterConfig: {}", config);
-
- return config;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java
deleted file mode 100755
index 16e4e06..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/GuiceServletConfig.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.listeners;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.TypeLiteral;
-import com.google.inject.matcher.Matchers;
-import com.google.inject.servlet.GuiceServletContextListener;
-import com.google.inject.spi.TypeEncounter;
-import com.google.inject.spi.TypeListener;
-import com.sun.jersey.api.core.PackagesResourceConfig;
-import com.sun.jersey.guice.JerseyServletModule;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.apache.hadoop.metadata.RepositoryMetadataModule;
-import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.web.filters.AuditFilter;
-import org.apache.hadoop.metadata.web.filters.MetadataAuthenticationFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.ServletContextEvent;
-import java.util.HashMap;
-import java.util.Map;
-
-import static com.google.inject.matcher.Matchers.*;
-
-public class GuiceServletConfig extends GuiceServletContextListener {
-
- private static final Logger LOG = LoggerFactory.getLogger(GuiceServletConfig.class);
-
- private static final String GUICE_CTX_PARAM = "guice.packages";
- static final String HTTP_AUTHENTICATION_ENABLED = "metadata.http.authentication.enabled";
- private Injector injector;
-
- @Override
- protected Injector getInjector() {
- LOG.info("Loading Guice modules");
- /*
- * More information on this can be found here:
- * https://jersey.java.net/nonav/apidocs/1
- * .11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary
- * .html
- */
- if (injector == null) {
- injector = Guice.createInjector(
- new RepositoryMetadataModule(),
- new JerseyServletModule() {
- @Override
- protected void configureServlets() {
- filter("/*").through(AuditFilter.class);
- try {
- configureAuthenticationFilter();
- } catch (ConfigurationException e) {
- LOG.warn("Unable to add and configure authentication filter", e);
- }
-
- String packages = getServletContext().getInitParameter(GUICE_CTX_PARAM);
-
- LOG.info("Jersey loading from packages: " + packages);
-
- Map<String, String> params = new HashMap<>();
- params.put(PackagesResourceConfig.PROPERTY_PACKAGES, packages);
- serve("/" + MetadataServiceClient.BASE_URI + "*").with(GuiceContainer.class, params);
- }
-
- private void configureAuthenticationFilter() throws ConfigurationException {
- try {
- PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
- if (Boolean.valueOf(configuration.getString(HTTP_AUTHENTICATION_ENABLED))) {
- filter("/*").through(MetadataAuthenticationFilter.class);
- }
- } catch (MetadataException e) {
- LOG.warn("Error loading configuration and initializing authentication filter", e);
- }
- }
- });
-
- LOG.info("Guice modules loaded");
- }
-
- return injector;
- }
-
- @Override
- public void contextInitialized(ServletContextEvent servletContextEvent) {
- super.contextInitialized(servletContextEvent);
-
- // perform login operations
- LoginProcessor loginProcessor = new LoginProcessor();
- loginProcessor.login();
- }
-
- @Override
- public void contextDestroyed(ServletContextEvent servletContextEvent) {
- super.contextDestroyed(servletContextEvent);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/LoginProcessor.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/LoginProcessor.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/LoginProcessor.java
deleted file mode 100644
index 271eae4..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/listeners/LoginProcessor.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.listeners;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.apache.hadoop.metadata.security.SecurityProperties;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.ServletException;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-/**
- * A class capable of performing a simple or kerberos login.
- */
-public class LoginProcessor {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(LoginProcessor.class);
- public static final String METADATA_AUTHENTICATION_PREFIX = "metadata.authentication.";
- public static final String AUTHENTICATION_METHOD = METADATA_AUTHENTICATION_PREFIX + "method";
- public static final String AUTHENTICATION_PRINCIPAL = METADATA_AUTHENTICATION_PREFIX + "principal";
- public static final String AUTHENTICATION_KEYTAB = METADATA_AUTHENTICATION_PREFIX + "keytab";
-
- /**
- * Perform a SIMPLE login based on established OS identity or a kerberos based login using the configured
- * principal and keytab (via application.properties).
- */
- public void login() {
- // first, let's see if we're running in a hadoop cluster and have the env configured
- boolean isHadoopCluster = isHadoopCluster();
- Configuration hadoopConfig = isHadoopCluster ? getHadoopConfiguration() : new Configuration(false);
- PropertiesConfiguration configuration = null;
- try {
- configuration = getPropertiesConfiguration();
- } catch (ConfigurationException e) {
- LOG.warn("Error reading application configuration", e);
- }
- if (!isHadoopCluster) {
- // need to read the configured authentication choice and create the UGI configuration
- setupHadoopConfiguration(hadoopConfig, configuration);
- }
- doServiceLogin(hadoopConfig, configuration);
- }
-
- protected void doServiceLogin(Configuration hadoopConfig, PropertiesConfiguration configuration) {
- UserGroupInformation.setConfiguration(hadoopConfig);
-
- UserGroupInformation ugi = null;
- UserGroupInformation.AuthenticationMethod authenticationMethod =
- SecurityUtil.getAuthenticationMethod(hadoopConfig);
- try {
- if (authenticationMethod == UserGroupInformation.AuthenticationMethod.SIMPLE) {
- UserGroupInformation.loginUserFromSubject(null);
- } else if (authenticationMethod == UserGroupInformation.AuthenticationMethod.KERBEROS) {
- String bindAddress = getHostname(configuration);
- UserGroupInformation.loginUserFromKeytab(
- getServerPrincipal(configuration.getString(AUTHENTICATION_PRINCIPAL), bindAddress),
- configuration.getString(AUTHENTICATION_KEYTAB));
- }
- LOG.info("Logged in user {}", UserGroupInformation.getLoginUser());
- } catch (IOException e) {
- throw new IllegalStateException(String.format("Unable to perform %s login.", authenticationMethod), e);
- }
- }
-
- private String getHostname(PropertiesConfiguration configuration) {
- String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS);
- if (bindAddress == null) {
- LOG.info("No host name configured. Defaulting to local host name.");
- try {
- bindAddress = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- throw new IllegalStateException(e);
- }
- }
- return bindAddress;
- }
-
- protected void setupHadoopConfiguration(Configuration hadoopConfig, PropertiesConfiguration configuration) {
- String authMethod;
- authMethod = configuration != null ? configuration.getString(AUTHENTICATION_METHOD) : null;
- // getString may return null, and would like to log the nature of the default setting
- if (authMethod == null) {
- LOG.info("No authentication method configured. Defaulting to simple authentication");
- authMethod = "simple";
- }
- SecurityUtil.setAuthenticationMethod(
- UserGroupInformation.AuthenticationMethod.valueOf(authMethod.toUpperCase()),
- hadoopConfig);
- }
-
- /**
- * Return a server (service) principal. The token "_HOST" in the principal will be replaced with the local host
- * name (e.g. dgi/_HOST will be changed to dgi/localHostName)
- * @param principal the input principal containing an option "_HOST" token
- * @return the service principal.
- * @throws IOException
- */
- private String getServerPrincipal(String principal, String host) throws IOException {
- return SecurityUtil.getServerPrincipal(principal, host);
- }
-
- /**
- * Returns a Hadoop configuration instance.
- * @return the configuration.
- */
- protected Configuration getHadoopConfiguration() {
- return new Configuration();
- }
-
- /**
- * Returns the metadata application configuration.
- * @return the metadata configuration.
- * @throws ConfigurationException
- */
- protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
- try {
- return PropertiesUtil.getApplicationProperties();
- } catch (MetadataException e) {
- throw new ConfigurationException(e);
- }
- }
-
- /**
- * Uses a hadoop shell to discern whether a hadoop cluster is available/configured.
- * @return true if a hadoop cluster is detected.
- */
- protected boolean isHadoopCluster() {
- boolean isHadoopCluster = false;
- try {
- isHadoopCluster = Shell.getHadoopHome() != null;
- } catch (IOException e) {
- // ignore - false is default setting
- }
- return isHadoopCluster;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/params/AbstractParam.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/params/AbstractParam.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/params/AbstractParam.java
deleted file mode 100755
index 7af7784..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/params/AbstractParam.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.params;
-
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-/**
- * An abstract base class from which to build Jersey parameter classes.
- *
- * @param <T> the type of value wrapped by the parameter
- */
-public abstract class AbstractParam<T> {
- private final T value;
-
- /**
- * Given an input value from a client, creates a parameter wrapping its parsed value.
- *
- * @param input an input value from a client request
- */
- @SuppressWarnings({"AbstractMethodCallInConstructor",
- "OverriddenMethodCallDuringObjectConstruction"})
- protected AbstractParam(String input) {
- try {
- this.value = parse(input);
- } catch (Exception e) {
- throw new WebApplicationException(error(input, e));
- }
- }
-
- /**
- * Given a string representation which was unable to be parsed and the exception thrown, produce
- * a {@link javax.ws.rs.core.Response} to be sent to the client.
- *
- * By default, generates a {@code 400 Bad Request} with a plain text entity generated by
- * {@link #errorMessage(String, Exception)}.
- *
- * @param input the raw input value
- * @param e the exception thrown while parsing {@code input}
- * @return the {@link javax.ws.rs.core.Response} to be sent to the client
- */
- protected Response error(String input, Exception e) {
- return Response.status(getErrorStatus())
- .entity(errorMessage(input, e))
- .type(mediaType())
- .build();
- }
-
- /**
- * Returns the media type of the error message entity.
- *
- * @return the media type of the error message entity
- */
- protected MediaType mediaType() {
- return MediaType.TEXT_PLAIN_TYPE;
- }
-
- /**
- * Given a string representation which was unable to be parsed and the exception thrown, produce
- * an entity to be sent to the client.
- *
- * @param input the raw input value
- * @param e the exception thrown while parsing {@code input}
- * @return the error message to be sent the client
- */
- protected String errorMessage(String input, Exception e) {
- return String.format("Invalid parameter: %s (%s)", input, e.getMessage());
- }
-
- /**
- * Given a string representation which was unable to be parsed, produce a {@link javax.ws.rs
- * .core.Response.Status} for the
- * {@link Response} to be sent to the client.
- *
- * @return the HTTP {@link javax.ws.rs.core.Response.Status} of the error message
- */
- @SuppressWarnings("MethodMayBeStatic")
- protected Response.Status getErrorStatus() {
- return Response.Status.BAD_REQUEST;
- }
-
- /**
- * Given a string representation, parse it and return an instance of the parameter type.
- *
- * @param input the raw input
- * @return {@code input}, parsed as an instance of {@code T}
- * @throws Exception if there is an error parsing the input
- */
- protected abstract T parse(String input) throws Exception;
-
- /**
- * Returns the underlying value.
- *
- * @return the underlying value
- */
- public T get() {
- return value;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if ((obj == null) || (getClass() != obj.getClass())) {
- return false;
- }
- final AbstractParam<?> that = (AbstractParam<?>) obj;
- return value.equals(that.value);
- }
-
- @Override
- public int hashCode() {
- return value.hashCode();
- }
-
- @Override
- public String toString() {
- return value.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/params/BooleanParam.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/params/BooleanParam.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/params/BooleanParam.java
deleted file mode 100755
index 01f40fc..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/params/BooleanParam.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.params;
-
-/**
- * A parameter encapsulating boolean values. If the query parameter value is {@code "true"},
- * regardless of case, the returned value is {@link Boolean#TRUE}. If the query parameter value is
- * {@code "false"}, regardless of case, the returned value is {@link Boolean#FALSE}. All other
- * values will return a {@code 400 Bad Request} response.
- */
-public class BooleanParam extends AbstractParam<Boolean> {
-
- public BooleanParam(String input) {
- super(input);
- }
-
- @Override
- protected String errorMessage(String input, Exception e) {
- return '"' + input + "\" must be \"true\" or \"false\".";
- }
-
- @Override
- protected Boolean parse(String input) throws Exception {
- if ("true".equalsIgnoreCase(input)) {
- return Boolean.TRUE;
- }
- if ("false".equalsIgnoreCase(input)) {
- return Boolean.FALSE;
- }
- throw new Exception();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/params/DateTimeParam.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/params/DateTimeParam.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/params/DateTimeParam.java
deleted file mode 100755
index b40ed05..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/params/DateTimeParam.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.params;
-
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-
-/**
- * A parameter encapsulating date/time values. All non-parsable values will return a {@code 400 Bad
- * Request} response. All values returned are in UTC.
- */
-public class DateTimeParam extends AbstractParam<DateTime> {
-
- public DateTimeParam(String input) {
- super(input);
- }
-
- @Override
- protected DateTime parse(String input) throws Exception {
- return new DateTime(input, DateTimeZone.UTC);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/AdminResource.java
deleted file mode 100755
index 86f9c22..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/AdminResource.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-
-import javax.inject.Singleton;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-/**
- * Jersey Resource for admin operations.
- */
-@Path("admin")
-@Singleton
-public class AdminResource {
-
- private Response version;
-
- /**
- * Fetches the thread stack dump for this application.
- *
- * @return json representing the thread stack dump.
- */
- @GET
- @Path("stack")
- @Produces(MediaType.TEXT_PLAIN)
- public String getThreadDump() {
- ThreadGroup topThreadGroup = Thread.currentThread().getThreadGroup();
-
- while (topThreadGroup.getParent() != null) {
- topThreadGroup = topThreadGroup.getParent();
- }
- Thread[] threads = new Thread[topThreadGroup.activeCount()];
-
- int nr = topThreadGroup.enumerate(threads);
- StringBuilder builder = new StringBuilder();
- for (int i = 0; i < nr; i++) {
- builder.append(threads[i].getName()).append("\nState: ").
- append(threads[i].getState()).append("\n");
- String stackTrace = StringUtils.join(threads[i].getStackTrace(), "\n");
- builder.append(stackTrace);
- }
- return builder.toString();
- }
-
- /**
- * Fetches the version for this application.
- *
- * @return json representing the version.
- */
- @GET
- @Path("version")
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response getVersion() {
- if (version == null) {
- try {
- PropertiesConfiguration configProperties =
- new PropertiesConfiguration("metadata-buildinfo.properties");
-
- JSONObject response = new JSONObject();
- response.put("Version", configProperties.getString("build.version", "UNKNOWN"));
- response.put("Name",
- configProperties.getString("project.name", "metadata-governance"));
- response.put("Description", configProperties.getString("project.description",
- "Metadata Management and Data Governance Platform over Hadoop"));
-
- // todo: add hadoop version?
- // response.put("Hadoop", VersionInfo.getVersion() + "-r" + VersionInfo.getRevision());
- version = Response.ok(response).build();
- } catch (JSONException | ConfigurationException e) {
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- return version;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java
deleted file mode 100755
index 3d9162b..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/EntityResource.java
+++ /dev/null
@@ -1,355 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.ParamChecker;
-import org.apache.hadoop.metadata.repository.EntityNotFoundException;
-import org.apache.hadoop.metadata.services.MetadataService;
-import org.apache.hadoop.metadata.typesystem.types.ValueConversionException;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriBuilder;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.net.URI;
-import java.util.List;
-
-
-/**
- * Entity management operations as REST API.
- *
- * An entity is an "instance" of a Type. Entities conform to the definition
- * of the Type they correspond with.
- */
-@Path("entities")
-@Singleton
-public class EntityResource {
-
- private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
- private static final String TRAIT_NAME = "traitName";
-
- private final MetadataService metadataService;
-
- @Context
- UriInfo uriInfo;
-
- /**
- * Created by the Guice ServletModule and injected with the
- * configured MetadataService.
- *
- * @param metadataService metadata service handle
- */
- @Inject
- public EntityResource(MetadataService metadataService) {
- this.metadataService = metadataService;
- }
-
- /**
- * Submits an entity definition (instance) corresponding to a given type.
- */
- @POST
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response submit(@Context HttpServletRequest request) {
- try {
- final String entity = Servlets.getRequestPayload(request);
- LOG.debug("submitting entity {} ", entity);
-
- final String guid = metadataService.createEntity(entity);
-
- UriBuilder ub = uriInfo.getAbsolutePathBuilder();
- URI locationURI = ub.path(guid).build();
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.GUID, guid);
- response.put(MetadataServiceClient.DEFINITION,
- metadataService.getEntityDefinition(guid));
-
- return Response.created(locationURI).entity(response).build();
-
- } catch(ValueConversionException ve) {
- LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
- throw new WebApplicationException(
- Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
- } catch (MetadataException | IllegalArgumentException e) {
- LOG.error("Unable to persist entity instance", e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to persist entity instance", e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Fetch the complete definition of an entity given its GUID.
- *
- * @param guid GUID for the entity
- */
- @GET
- @Path("{guid}")
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response getEntityDefinition(@PathParam("guid") String guid) {
- try {
- LOG.debug("Fetching entity definition for guid={} ", guid);
- ParamChecker.notEmpty(guid, "guid cannot be null");
- final String entityDefinition = metadataService.getEntityDefinition(guid);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.GUID, guid);
-
- Response.Status status = Response.Status.NOT_FOUND;
- if (entityDefinition != null) {
- response.put(MetadataServiceClient.DEFINITION, entityDefinition);
- status = Response.Status.OK;
- } else {
- response.put(MetadataServiceClient.ERROR, Servlets.escapeJsonString(
- String.format("An entity with GUID={%s} does not exist", guid)));
- }
-
- return Response.status(status).entity(response).build();
-
- } catch (EntityNotFoundException e) {
- LOG.error("An entity with GUID={} does not exist", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (MetadataException | IllegalArgumentException e) {
- LOG.error("Bad GUID={}", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get instance definition for GUID {}", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Gets the list of entities for a given entity type.
- *
- * @param entityType name of a type which is unique
- */
- @GET
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response getEntityListByType(@QueryParam("type") String entityType) {
- try {
- Preconditions.checkNotNull(entityType, "Entity type cannot be null");
-
- LOG.debug("Fetching entity list for type={} ", entityType);
- final List<String> entityList = metadataService.getEntityList(entityType);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.TYPENAME, entityType);
- response.put(MetadataServiceClient.RESULTS, new JSONArray(entityList));
- response.put(MetadataServiceClient.COUNT, entityList.size());
-
- return Response.ok(response).build();
- } catch (NullPointerException e) {
- LOG.error("Entity type cannot be null", e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (MetadataException | IllegalArgumentException e) {
- LOG.error("Unable to get entity list for type {}", entityType, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get entity list for type {}", entityType, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Adds property to the given entity id
- * @param guid entity id
- * @param property property to add
- * @param value property's value
- * @return response payload as json
- */
- @PUT
- @Path("{guid}")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response update(@PathParam("guid") String guid,
- @QueryParam("property") String property,
- @QueryParam("value") String value) {
- try {
- Preconditions.checkNotNull(property, "Entity property cannot be null");
- Preconditions.checkNotNull(value, "Entity value cannot be null");
-
- metadataService.updateEntity(guid, property, value);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Thread.currentThread().getName());
- return Response.ok(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("An entity with GUID={} does not exist", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (MetadataException | IllegalArgumentException e) {
- LOG.error("Unable to add property {} to entity id {}", property, guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to add property {} to entity id {}", property, guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- // Trait management functions
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- */
- @GET
- @Path("{guid}/traits")
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response getTraitNames(@PathParam("guid") String guid) {
- try {
- LOG.debug("Fetching trait names for entity={}", guid);
- final List<String> traitNames = metadataService.getTraitNames(guid);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.GUID, guid);
- response.put(MetadataServiceClient.RESULTS, new JSONArray(traitNames));
- response.put(MetadataServiceClient.COUNT, traitNames.size());
-
- return Response.ok(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("An entity with GUID={} does not exist", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (MetadataException | IllegalArgumentException e) {
- LOG.error("Unable to get trait names for entity {}", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get trait names for entity {}", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- */
- @POST
- @Path("{guid}/traits")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response addTrait(@Context HttpServletRequest request,
- @PathParam("guid") String guid) {
- try {
- final String traitDefinition = Servlets.getRequestPayload(request);
- LOG.debug("Adding trait={} for entity={} ", traitDefinition, guid);
- metadataService.addTrait(guid, traitDefinition);
-
- UriBuilder ub = uriInfo.getAbsolutePathBuilder();
- URI locationURI = ub.path(guid).build();
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.GUID, guid);
-
- return Response.created(locationURI).entity(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("An entity with GUID={} does not exist", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (MetadataException | IOException | IllegalArgumentException e) {
- LOG.error("Unable to add trait for entity={}", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to add trait for entity={}", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitName name of the trait
- */
- @DELETE
- @Path("{guid}/traits/{traitName}")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response deleteTrait(@Context HttpServletRequest request,
- @PathParam("guid") String guid,
- @PathParam(TRAIT_NAME) String traitName) {
- LOG.debug("Deleting trait={} from entity={} ", traitName, guid);
- try {
- metadataService.deleteTrait(guid, traitName);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.GUID, guid);
- response.put(TRAIT_NAME, traitName);
-
- return Response.ok(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("An entity with GUID={} does not exist", guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (MetadataException | IllegalArgumentException e) {
- LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java
deleted file mode 100644
index 076a916..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/HiveLineageResource.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.ParamChecker;
-import org.apache.hadoop.metadata.discovery.DiscoveryException;
-import org.apache.hadoop.metadata.discovery.LineageService;
-import org.apache.hadoop.metadata.repository.EntityNotFoundException;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-
-/**
- * Jersey Resource for Hive Table Lineage.
- */
-@Path("lineage/hive")
-@Singleton
-public class HiveLineageResource {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveLineageResource.class);
-
- private final LineageService lineageService;
-
- /**
- * Created by the Guice ServletModule and injected with the
- * configured LineageService.
- *
- * @param lineageService lineage service handle
- */
- @Inject
- public HiveLineageResource(LineageService lineageService) {
- this.lineageService = lineageService;
- }
-
- /**
- * Returns the inputs graph for a given entity.
- *
- * @param tableName table name
- */
- @GET
- @Path("table/{tableName}/inputs/graph")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response inputsGraph(@Context HttpServletRequest request,
- @PathParam("tableName") String tableName) {
- LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
-
- try {
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- final String jsonResult = lineageService.getInputsGraph(tableName);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put("tableName", tableName);
- response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
-
- return Response.ok(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("table entity not found for {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (DiscoveryException | IllegalArgumentException e) {
- LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get lineage inputs graph for table {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Returns the outputs graph for a given entity.
- *
- * @param tableName table name
- */
- @GET
- @Path("table/{tableName}/outputs/graph")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response outputsGraph(@Context HttpServletRequest request,
- @PathParam("tableName") String tableName) {
- LOG.info("Fetching lineage outputs graph for tableName={}", tableName);
-
- try {
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- final String jsonResult = lineageService.getOutputsGraph(tableName);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put("tableName", tableName);
- response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
-
- return Response.ok(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("table entity not found for {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (DiscoveryException | IllegalArgumentException e) {
- LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get lineage outputs graph for table {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Return the schema for the given tableName.
- *
- * @param tableName table name
- */
- @GET
- @Path("table/{tableName}/schema")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response schema(@Context HttpServletRequest request,
- @PathParam("tableName") String tableName) {
- LOG.info("Fetching schema for tableName={}", tableName);
-
- try {
- ParamChecker.notEmpty(tableName, "table name cannot be null");
- final String jsonResult = lineageService.getSchema(tableName);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put("tableName", tableName);
- response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult));
-
- return Response.ok(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("table entity not found for {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (DiscoveryException | IllegalArgumentException e) {
- LOG.error("Unable to get schema for table {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to get schema for table {}", tableName, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java
deleted file mode 100755
index 5928e37..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/MetadataDiscoveryResource.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.ParamChecker;
-import org.apache.hadoop.metadata.discovery.DiscoveryException;
-import org.apache.hadoop.metadata.discovery.DiscoveryService;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Jersey Resource for metadata operations.
- */
-@Path("discovery")
-@Singleton
-public class MetadataDiscoveryResource {
-
- private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class);
- private static final String QUERY_TYPE_DSL = "dsl";
- private static final String QUERY_TYPE_GREMLIN = "gremlin";
- private static final String QUERY_TYPE_FULLTEXT = "full-text";
-
- private final DiscoveryService discoveryService;
-
- /**
- * Created by the Guice ServletModule and injected with the
- * configured DiscoveryService.
- *
- * @param discoveryService metadata service handle
- */
- @Inject
- public MetadataDiscoveryResource(DiscoveryService discoveryService) {
- this.discoveryService = discoveryService;
- }
-
- /**
- * Search using a given query.
- *
- * @param query search query in raw gremlin or DSL format falling back to full text.
- * @return JSON representing the type and results.
- */
- @GET
- @Path("search")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response search(@QueryParam("query") String query) {
- JSONObject response;
- try { // fall back to dsl
- ParamChecker.notEmpty(query, "query cannot be null");
-
- if (query.startsWith("g.")) { // raw gremlin query
- return searchUsingGremlinQuery(query);
- }
-
- final String jsonResultStr = discoveryService.searchByDSL(query);
- response = new DSLJSONResponseBuilder().results(jsonResultStr)
- .query(query)
- .build();
-
- } catch (IllegalArgumentException e) {
- LOG.error("Unable to get entity list for empty query", e);
- throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable throwable) {
- LOG.error("Unable to get entity list for query {} using dsl", query, throwable);
-
- try { //fall back to full-text
- final String jsonResultStr = discoveryService.searchByFullText(query);
- response = new FullTextJSonResponseBuilder().results(jsonResultStr)
- .query(query)
- .build();
-
- } catch (DiscoveryException | IllegalArgumentException e) {
- LOG.error("Unable to get entity list for query {}", query, e);
- throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch(Throwable e) {
- LOG.error("Unable to get entity list for query {}", query, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- return Response.ok(response)
- .build();
-
- }
-
- /**
- * Search using query DSL format.
- *
- * @param dslQuery search query in DSL format.
- * @return JSON representing the type and results.
- */
- @GET
- @Path("search/dsl")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response searchUsingQueryDSL(@QueryParam("query") String dslQuery) {
- try {
- ParamChecker.notEmpty(dslQuery, "dslQuery cannot be null");
- final String jsonResultStr = discoveryService.searchByDSL(dslQuery);
-
- JSONObject response = new DSLJSONResponseBuilder().results(jsonResultStr)
- .query(dslQuery)
- .build();
-
- return Response.ok(response)
- .build();
- } catch (DiscoveryException | IllegalArgumentException e) {
- LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch(Throwable e) {
- LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Search using raw gremlin query format.
- *
- * @param gremlinQuery search query in raw gremlin format.
- * @return JSON representing the type and results.
- */
- @GET
- @Path("search/gremlin")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response searchUsingGremlinQuery(@QueryParam("query") String gremlinQuery) {
- try {
- ParamChecker.notEmpty(gremlinQuery, "gremlinQuery cannot be null or empty");
- final List<Map<String, String>> results = discoveryService
- .searchByGremlin(gremlinQuery);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.QUERY, gremlinQuery);
- response.put(MetadataServiceClient.QUERY_TYPE, QUERY_TYPE_GREMLIN);
-
- JSONArray list = new JSONArray();
- for (Map<String, String> result : results) {
- list.put(new JSONObject(result));
- }
- response.put(MetadataServiceClient.RESULTS, list);
- response.put(MetadataServiceClient.COUNT, list.length());
-
- return Response.ok(response)
- .build();
- } catch (DiscoveryException | IllegalArgumentException e) {
- LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch(Throwable e) {
- LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e);
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Search using full text search.
- *
- * @param query search query.
- * @return JSON representing the type and results.
- */
- @GET
- @Path("search/fulltext")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response searchUsingFullText(@QueryParam("query") String query) {
- try {
- ParamChecker.notEmpty(query, "query cannot be null or empty");
- final String jsonResultStr = discoveryService.searchByFullText(query);
- JSONArray rowsJsonArr = new JSONArray(jsonResultStr);
-
- JSONObject response = new FullTextJSonResponseBuilder().results(rowsJsonArr)
- .query(query)
- .build();
- return Response.ok(response)
- .build();
- } catch (DiscoveryException | IllegalArgumentException e) {
- LOG.error("Unable to get entity list for query {}", query, e);
- throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch(Throwable e) {
- LOG.error("Unable to get entity list for query {}", query, e);
- throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- private class JsonResponseBuilder {
-
- protected int count = 0;
- protected String query;
- protected String queryType;
- protected JSONObject response;
-
- JsonResponseBuilder() {
- this.response = new JSONObject();
- }
-
- protected JsonResponseBuilder count(int count) {
- this.count = count;
- return this;
- }
-
- public JsonResponseBuilder query(String query) {
- this.query = query;
- return this;
- }
-
- public JsonResponseBuilder queryType(String queryType) {
- this.queryType = queryType;
- return this;
- }
-
- protected JSONObject build() throws JSONException {
-
- Preconditions.checkNotNull(query, "Query cannot be null");
- Preconditions.checkNotNull(queryType, "Query Type must be specified");
- Preconditions.checkArgument(count >= 0, "Search Result count should be > 0");
-
- response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId());
- response.put(MetadataServiceClient.QUERY, query);
- response.put(MetadataServiceClient.QUERY_TYPE, queryType);
- response.put(MetadataServiceClient.COUNT, count);
- return response;
- }
- }
-
- private class DSLJSONResponseBuilder extends JsonResponseBuilder {
-
- DSLJSONResponseBuilder() {
- super();
- }
-
- private JSONObject dslResults;
-
- public DSLJSONResponseBuilder results(JSONObject dslResults) {
- this.dslResults = dslResults;
- return this;
- }
-
- public DSLJSONResponseBuilder results(String dslResults) throws JSONException {
- return results(new JSONObject(dslResults));
- }
-
- @Override
- public JSONObject build() throws JSONException {
- Preconditions.checkNotNull(dslResults);
- JSONArray rowsJsonArr = dslResults.getJSONArray(MetadataServiceClient.ROWS);
- count(rowsJsonArr.length());
- queryType(QUERY_TYPE_DSL);
- JSONObject response = super.build();
- response.put(MetadataServiceClient.RESULTS, dslResults);
- return response;
- }
-
- }
-
- private class FullTextJSonResponseBuilder extends JsonResponseBuilder {
-
- private JSONArray fullTextResults;
-
- public FullTextJSonResponseBuilder results(JSONArray fullTextResults) {
- this.fullTextResults = fullTextResults;
- return this;
- }
-
- public FullTextJSonResponseBuilder results(String dslResults) throws JSONException {
- return results(new JSONArray(dslResults));
- }
-
- public FullTextJSonResponseBuilder() {
- super();
- }
-
- @Override
- public JSONObject build() throws JSONException {
- Preconditions.checkNotNull(fullTextResults);
- count(fullTextResults.length());
- queryType(QUERY_TYPE_FULLTEXT);
-
- JSONObject response = super.build();
- response.put(MetadataServiceClient.RESULTS, fullTextResults);
- return response;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java b/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java
deleted file mode 100755
index 0092879..0000000
--- a/webapp/src/main/java/org/apache/hadoop/metadata/web/resources/RexsterGraphResource.java
+++ /dev/null
@@ -1,408 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Element;
-import com.tinkerpop.blueprints.Graph;
-import com.tinkerpop.blueprints.Vertex;
-import com.tinkerpop.blueprints.VertexQuery;
-import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
-import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Jersey Resource for lineage metadata operations.
- * Implements most of the GET operations of Rexster API with out the indexes.
- * https://github.com/tinkerpop/rexster/wiki/Basic-REST-API
- *
- * This is a subset of Rexster's REST API, designed to provide only read-only methods
- * for accessing the backend graph.
- */
-@Path("graph")
-@Singleton
-public class RexsterGraphResource {
- public static final String OUT_E = "outE";
- public static final String IN_E = "inE";
- public static final String BOTH_E = "bothE";
- public static final String OUT = "out";
- public static final String IN = "in";
- public static final String BOTH = "both";
- public static final String OUT_COUNT = "outCount";
- public static final String IN_COUNT = "inCount";
- public static final String BOTH_COUNT = "bothCount";
- public static final String OUT_IDS = "outIds";
- public static final String IN_IDS = "inIds";
- public static final String BOTH_IDS = "bothIds";
- private static final Logger LOG = LoggerFactory.getLogger(RexsterGraphResource.class);
-
- private TitanGraph graph;
-
- @Inject
- public RexsterGraphResource(GraphProvider<TitanGraph> graphProvider) {
- this.graph = graphProvider.get();
- }
-
- private static void validateInputs(String errorMsg, String... inputs) {
- for (String input : inputs) {
- if (StringUtils.isEmpty(input)) {
- throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
- .entity(errorMsg)
- .type("text/plain")
- .build());
- }
- }
- }
-
- protected Graph getGraph() {
- return graph;
- }
-
- protected Set<String> getVertexIndexedKeys() {
- return graph.getIndexedKeys(Vertex.class);
- }
-
- protected Set<String> getEdgeIndexedKeys() {
- return graph.getIndexedKeys(Edge.class);
- }
-
- /**
- * Get a single vertex with a unique id.
- *
- * GET http://host/metadata/lineage/vertices/id
- * graph.getVertex(id);
- */
- @GET
- @Path("/vertices/{id}")
- @Produces({Servlets.JSON_MEDIA_TYPE})
- public Response getVertex(@PathParam("id") final String vertexId) {
- LOG.info("Get vertex for vertexId= {}", vertexId);
- validateInputs("Invalid argument: vertex id passed is null or empty.", vertexId);
- try {
- Vertex vertex = findVertex(vertexId);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.RESULTS, GraphSONUtility.jsonFromElement(
- vertex, getVertexIndexedKeys(), GraphSONMode.NORMAL));
- return Response.ok(response).build();
- } catch (JSONException e) {
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- private Vertex findVertex(String vertexId) {
- Vertex vertex = getGraph().getVertex(vertexId);
- if (vertex == null) {
- String message = "Vertex with [" + vertexId + "] cannot be found.";
- LOG.info(message);
- throw new WebApplicationException(
- Servlets.getErrorResponse(message, Response.Status.NOT_FOUND));
- }
-
- return vertex;
- }
-
- /**
- * Get properties for a single vertex with a unique id.
- * This is NOT a rexster API.
- * <p/>
- * GET http://host/metadata/lineage/vertices/properties/id
- */
- @GET
- @Path("/vertices/properties/{id}")
- @Produces({Servlets.JSON_MEDIA_TYPE})
- public Response getVertexProperties(@PathParam("id") final String vertexId,
- @DefaultValue("false") @QueryParam("relationships")
- final String relationships) {
- LOG.info("Get vertex for vertexId= {}", vertexId);
- validateInputs("Invalid argument: vertex id passed is null or empty.", vertexId);
- try {
- Vertex vertex = findVertex(vertexId);
-
- Map<String, String> vertexProperties = getVertexProperties(vertex);
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.RESULTS, new JSONObject(vertexProperties));
- response.put(MetadataServiceClient.COUNT, vertexProperties.size());
- return Response.ok(response).build();
- } catch (JSONException e) {
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- private Map<String, String> getVertexProperties(Vertex vertex) {
- Map<String, String> vertexProperties = new HashMap<>();
- for (String key : vertex.getPropertyKeys()) {
- vertexProperties.put(key, vertex.<String>getProperty(key));
- }
-
- // todo: get the properties from relationships
-
- return vertexProperties;
- }
-
- /**
- * Get a list of vertices matching a property key and a value.
- * <p/>
- * GET http://host/metadata/lineage/vertices?key=<key>&value=<value>
- * graph.getVertices(key, value);
- */
- @GET
- @Path("/vertices")
- @Produces({Servlets.JSON_MEDIA_TYPE})
- public Response getVertices(@QueryParam("key") final String key,
- @QueryParam("value") final String value) {
- LOG.info("Get vertices for property key= {}, value= {}", key, value);
- validateInputs("Invalid argument: key or value passed is null or empty.", key, value);
- try {
- JSONObject response = buildJSONResponse(getGraph().getVertices(key, value));
- return Response.ok(response).build();
-
- } catch (JSONException e) {
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- /**
- * Get a list of adjacent edges with a direction.
- *
- * GET http://host/metadata/lineage/vertices/id/direction
- * graph.getVertex(id).get{Direction}Edges();
- * direction: {(?!outE)(?!bothE)(?!inE)(?!out)(?!both)(?!in)(?!query).+}
- */
- @GET
- @Path("vertices/{id}/{direction}")
- @Produces({Servlets.JSON_MEDIA_TYPE})
- public Response getVertexEdges(@PathParam("id") String vertexId,
- @PathParam("direction") String direction) {
- LOG.info("Get vertex edges for vertexId= {}, direction= {}", vertexId, direction);
- // Validate vertex id. Direction is validated in VertexQueryArguments.
- validateInputs("Invalid argument: vertex id or direction passed is null or empty.",
- vertexId, direction);
- try {
- Vertex vertex = findVertex(vertexId);
-
- return getVertexEdges(vertex, direction);
-
- } catch (JSONException e) {
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- private Response getVertexEdges(Vertex vertex, String direction) throws JSONException {
- // break out the segment into the return and the direction
- VertexQueryArguments queryArguments = new VertexQueryArguments(direction);
- // if this is a query and the _return is "count" then we don't bother to send back the
- // result array
- boolean countOnly = queryArguments.isCountOnly();
- // what kind of data the calling client wants back (vertices, edges, count, vertex
- // identifiers)
- ReturnType returnType = queryArguments.getReturnType();
- // the query direction (both, out, in)
- Direction queryDirection = queryArguments.getQueryDirection();
-
- VertexQuery query = vertex.query().direction(queryDirection);
-
- JSONArray elementArray = new JSONArray();
- long counter = 0;
- if (returnType == ReturnType.VERTICES || returnType == ReturnType.VERTEX_IDS) {
- Iterable<Vertex> vertexQueryResults = query.vertices();
- for (Vertex v : vertexQueryResults) {
- if (returnType.equals(ReturnType.VERTICES)) {
- elementArray.put(GraphSONUtility.jsonFromElement(
- v, getVertexIndexedKeys(), GraphSONMode.NORMAL));
- } else {
- elementArray.put(v.getId());
- }
- counter++;
- }
- } else if (returnType == ReturnType.EDGES) {
- Iterable<Edge> edgeQueryResults = query.edges();
- for (Edge e : edgeQueryResults) {
- elementArray.put(GraphSONUtility.jsonFromElement(
- e, getEdgeIndexedKeys(), GraphSONMode.NORMAL));
- counter++;
- }
- } else if (returnType == ReturnType.COUNT) {
- counter = query.count();
- }
-
- JSONObject response = new JSONObject();
- if (!countOnly) {
- response.put(MetadataServiceClient.RESULTS, elementArray);
- }
- response.put(MetadataServiceClient.COUNT, counter);
- return Response.ok(response).build();
- }
-
- /**
- * Get a single edge with a unique id.
- *
- * GET http://host/metadata/lineage/edges/id
- * graph.getEdge(id);
- */
- @GET
- @Path("/edges/{id}")
- @Produces({Servlets.JSON_MEDIA_TYPE})
- public Response getEdge(@PathParam("id") final String edgeId) {
- LOG.info("Get vertex for edgeId= {}", edgeId);
- validateInputs("Invalid argument: edge id passed is null or empty.", edgeId);
- try {
- Edge edge = getGraph().getEdge(edgeId);
- if (edge == null) {
- String message = "Edge with [" + edgeId + "] cannot be found.";
- LOG.info(message);
- throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND)
- .entity(Servlets.escapeJsonString(message)).build());
- }
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.RESULTS, GraphSONUtility.jsonFromElement(
- edge, getEdgeIndexedKeys(), GraphSONMode.NORMAL));
- return Response.ok(response).build();
- } catch (JSONException e) {
- throw new WebApplicationException(
- Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
-
- private <T extends Element> JSONObject buildJSONResponse(Iterable<T> elements)
- throws JSONException {
- JSONArray vertexArray = new JSONArray();
- long counter = 0;
- for (Element element : elements) {
- counter++;
- vertexArray.put(GraphSONUtility.jsonFromElement(
- element, getVertexIndexedKeys(), GraphSONMode.NORMAL));
- }
-
- JSONObject response = new JSONObject();
- response.put(MetadataServiceClient.RESULTS, vertexArray);
- response.put(MetadataServiceClient.COUNT, counter);
-
- return response;
- }
- private enum ReturnType {VERTICES, EDGES, COUNT, VERTEX_IDS}
-
- /**
- * Helper class for query arguments.
- */
- public static final class VertexQueryArguments {
-
- private final Direction queryDirection;
- private final ReturnType returnType;
- private final boolean countOnly;
-
- public VertexQueryArguments(String directionSegment) {
- if (OUT_E.equals(directionSegment)) {
- returnType = ReturnType.EDGES;
- queryDirection = Direction.OUT;
- countOnly = false;
- } else if (IN_E.equals(directionSegment)) {
- returnType = ReturnType.EDGES;
- queryDirection = Direction.IN;
- countOnly = false;
- } else if (BOTH_E.equals(directionSegment)) {
- returnType = ReturnType.EDGES;
- queryDirection = Direction.BOTH;
- countOnly = false;
- } else if (OUT.equals(directionSegment)) {
- returnType = ReturnType.VERTICES;
- queryDirection = Direction.OUT;
- countOnly = false;
- } else if (IN.equals(directionSegment)) {
- returnType = ReturnType.VERTICES;
- queryDirection = Direction.IN;
- countOnly = false;
- } else if (BOTH.equals(directionSegment)) {
- returnType = ReturnType.VERTICES;
- queryDirection = Direction.BOTH;
- countOnly = false;
- } else if (BOTH_COUNT.equals(directionSegment)) {
- returnType = ReturnType.COUNT;
- queryDirection = Direction.BOTH;
- countOnly = true;
- } else if (IN_COUNT.equals(directionSegment)) {
- returnType = ReturnType.COUNT;
- queryDirection = Direction.IN;
- countOnly = true;
- } else if (OUT_COUNT.equals(directionSegment)) {
- returnType = ReturnType.COUNT;
- queryDirection = Direction.OUT;
- countOnly = true;
- } else if (BOTH_IDS.equals(directionSegment)) {
- returnType = ReturnType.VERTEX_IDS;
- queryDirection = Direction.BOTH;
- countOnly = false;
- } else if (IN_IDS.equals(directionSegment)) {
- returnType = ReturnType.VERTEX_IDS;
- queryDirection = Direction.IN;
- countOnly = false;
- } else if (OUT_IDS.equals(directionSegment)) {
- returnType = ReturnType.VERTEX_IDS;
- queryDirection = Direction.OUT;
- countOnly = false;
- } else {
- throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
- .entity(Servlets.escapeJsonString(directionSegment + " segment was invalid."))
- .build());
- }
- }
-
- public Direction getQueryDirection() {
- return queryDirection;
- }
-
- public ReturnType getReturnType() {
- return returnType;
- }
-
- public boolean isCountOnly() {
- return countOnly;
- }
- }
-}
[25/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
new file mode 100755
index 0000000..56df843
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import junit.framework.Assert;
+import org.apache.atlas.GraphTransaction;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.List;
+
+@Guice(modules = RepositoryMetadataModule.class)
+public class GraphBackedTypeStoreTest {
+ @Inject
+ private GraphProvider<TitanGraph> graphProvider;
+
+ @Inject
+ private ITypeStore typeStore;
+
+ private TypeSystem ts;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ ts = TypeSystem.getInstance();
+ ts.reset();
+ TestUtils.defineDeptEmployeeTypes(ts);
+ }
+
+ @Test
+ @GraphTransaction
+ public void testStore() throws MetadataException {
+ typeStore.store(ts);
+ dumpGraph();
+ }
+
+ private void dumpGraph() {
+ TitanGraph graph = graphProvider.get();
+ for (Vertex v : graph.getVertices()) {
+ System.out.println("****v = " + GraphHelper.vertexString(v));
+ for (Edge e : v.getEdges(Direction.OUT)) {
+ System.out.println("****e = " + GraphHelper.edgeString(e));
+ }
+ }
+ }
+
+ @Test (dependsOnMethods = "testStore")
+ @GraphTransaction
+ public void testRestore() throws Exception {
+ TypesDef types = typeStore.restore();
+
+ //validate enum
+ List<EnumTypeDefinition> enumTypes = types.enumTypesAsJavaList();
+ Assert.assertEquals(1, enumTypes.size());
+ EnumTypeDefinition orgLevel = enumTypes.get(0);
+ Assert.assertEquals(orgLevel.name, "OrgLevel");
+ Assert.assertEquals(orgLevel.enumValues.length, 2);
+ EnumValue enumValue = orgLevel.enumValues[0];
+ Assert.assertEquals(enumValue.value, "L1");
+ Assert.assertEquals(enumValue.ordinal, 1);
+
+ //validate class
+ List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
+ Assert.assertEquals(1, structTypes.size());
+
+ boolean clsTypeFound = false;
+ List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
+ for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
+ if (classType.typeName.equals("Manager")) {
+ ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
+ Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
+ Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
+ clsTypeFound = true;
+ }
+ }
+ Assert.assertTrue("Manager type not restored", clsTypeFound);
+
+ //validate trait
+ List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList();
+ Assert.assertEquals(1, traitTypes.size());
+ HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
+ Assert.assertEquals("SecurityClearance", trait.typeName);
+ Assert.assertEquals(1, trait.attributeDefinitions.length);
+ AttributeDefinition attribute = trait.attributeDefinitions[0];
+ Assert.assertEquals("level", attribute.name);
+ Assert.assertEquals(DataTypes.INT_TYPE.getName(), attribute.dataTypeName);
+
+ //validate the new types
+ ts.reset();
+ ts.defineTypes(types);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/RepositoryServiceLoadingTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/hadoop/metadata/RepositoryServiceLoadingTest.java
deleted file mode 100755
index e5130fb..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/RepositoryServiceLoadingTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import com.thinkaurelius.titan.core.TitanGraph;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.testng.Assert;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-
-/**
- * Unit test for Guice injector service loading
- *
- * Uses TestNG's Guice annotation to load the necessary modules and inject the
- * objects from Guice
- */
-@Guice(modules = RepositoryMetadataModule.class)
-public class RepositoryServiceLoadingTest {
-
- @Inject
- private GraphProvider<TitanGraph> graphProvider;
-
- @Test
- public void testGetGraphService() throws Exception {
- Assert.assertNotNull(graphProvider);
- Assert.assertNotNull(graphProvider.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/TestUtils.java b/repository/src/test/java/org/apache/hadoop/metadata/TestUtils.java
deleted file mode 100755
index 8e2f82e..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/TestUtils.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata;
-
-import com.google.common.collect.ImmutableList;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
-import org.apache.hadoop.metadata.repository.graph.GraphHelper;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumType;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.testng.Assert;
-
-import java.io.File;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createStructTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-
-/**
- * Test utility class.
- */
-public final class TestUtils {
-
- private TestUtils() {
- }
-
- /**
- * Dumps the graph in GSON format in the path returned.
- *
- * @param titanGraph handle to graph
- * @return path to the dump file
- * @throws Exception
- */
- public static String dumpGraph(TitanGraph titanGraph) throws Exception {
- File tempFile = File.createTempFile("graph", ".gson");
- System.out.println("tempFile.getPath() = " + tempFile.getPath());
- GraphSONWriter.outputGraph(titanGraph, tempFile.getPath());
-
- System.out.println("Vertices:");
- for (Vertex vertex : titanGraph.getVertices()) {
- System.out.println(GraphHelper.vertexString(vertex));
- }
-
- System.out.println("Edges:");
- for (Edge edge : titanGraph.getEdges()) {
- System.out.println(GraphHelper.edgeString(edge));
- }
-
- return tempFile.getPath();
- }
-
- /**
- * Class Hierarchy is:
- * Department(name : String, employees : Array[Person])
- * Person(name : String, department : Department, manager : Manager)
- * Manager(subordinates : Array[Person]) extends Person
- * <p/>
- * Persons can have SecurityClearance(level : Int) clearance.
- */
- public static void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
-
- EnumTypeDefinition orgLevelEnum =
- new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1), new EnumValue("L2", 2));
- ts.defineEnumType(orgLevelEnum);
-
- StructTypeDefinition addressDetails = createStructTypeDef("Address",
- createRequiredAttrDef("street", DataTypes.STRING_TYPE),
- createRequiredAttrDef("city", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<ClassType> deptTypeDef =
- createClassTypeDef("Department", ImmutableList.<String>of(),
- createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees",
- String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true,
- "department")
- );
-
- HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person",
- ImmutableList.<String>of(),
- createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- createOptionalAttrDef("orgLevel", ts.getDataType(EnumType.class, "OrgLevel")),
- createOptionalAttrDef("address", "Address"),
- new AttributeDefinition("department",
- "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager",
- "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- );
-
- HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
- ImmutableList.of("Person"),
- new AttributeDefinition("subordinates",
- String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false,
- "manager")
- );
-
- HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef(
- "SecurityClearance",
- ImmutableList.<String>of(),
- createRequiredAttrDef("level", DataTypes.INT_TYPE)
- );
-
- ts.defineTypes(ImmutableList.of(addressDetails),
- ImmutableList.of(securityClearanceTypeDef),
- ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef));
- }
-
- public static Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
- Referenceable hrDept = new Referenceable("Department");
- Referenceable john = new Referenceable("Person");
- Referenceable jane = new Referenceable("Manager", "SecurityClearance");
- Referenceable johnAddr = new Referenceable("Address");
- Referenceable janeAddr = new Referenceable("Address");
-
- hrDept.set("name", "hr");
- john.set("name", "John");
- john.set("department", hrDept);
- johnAddr.set("street", "Stewart Drive");
- johnAddr.set("city", "Sunnyvale");
- john.set("address", johnAddr);
-
- jane.set("name", "Jane");
- jane.set("department", hrDept);
- janeAddr.set("street", "Great America Parkway");
- janeAddr.set("city", "Santa Clara");
- jane.set("address", janeAddr);
-
- john.set("manager", jane);
-
- hrDept.set("employees", ImmutableList.of(john, jane));
-
- jane.set("subordinates", ImmutableList.of(john));
-
- jane.getTrait("SecurityClearance").set("level", 1);
-
- ClassType deptType = ts.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
- Assert.assertNotNull(hrDept2);
-
- return hrDept;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
deleted file mode 100755
index 4d24c29..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/discovery/GraphBackedDiscoveryServiceTest.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery;
-
-import com.google.common.collect.ImmutableList;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.metadata.RepositoryMetadataModule;
-import org.apache.hadoop.metadata.TestUtils;
-import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.hadoop.metadata.query.HiveTitanSample;
-import org.apache.hadoop.metadata.query.QueryTestsUtils;
-import org.apache.hadoop.metadata.repository.graph.GraphBackedMetadataRepository;
-import org.apache.hadoop.metadata.repository.graph.GraphHelper;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-import javax.script.Bindings;
-import javax.script.ScriptEngine;
-import javax.script.ScriptEngineManager;
-import javax.script.ScriptException;
-import java.io.File;
-
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-
-@Guice(modules = RepositoryMetadataModule.class)
-public class GraphBackedDiscoveryServiceTest {
-
- @Inject
- private GraphProvider<TitanGraph> graphProvider;
-
- @Inject
- private GraphBackedMetadataRepository repositoryService;
-
- @Inject
- private GraphBackedDiscoveryService discoveryService;
-
- @BeforeClass
- public void setUp() throws Exception {
- TypeSystem typeSystem = TypeSystem.getInstance();
- typeSystem.reset();
-
- QueryTestsUtils.setupTypes();
- setupSampleData();
-
- TestUtils.defineDeptEmployeeTypes(typeSystem);
-
- Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
- ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
-
- repositoryService.createEntity(hrDept2);
- }
-
- private void setupSampleData() throws ScriptException {
- TitanGraph titanGraph = graphProvider.get();
-
- ScriptEngineManager manager = new ScriptEngineManager();
- ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
- Bindings bindings = engine.createBindings();
- bindings.put("g", titanGraph);
-
- String hiveGraphFile = FileUtils.getTempDirectory().getPath()
- + File.separator + System.nanoTime() + ".gson";
- System.out.println("hiveGraphFile = " + hiveGraphFile);
- HiveTitanSample.writeGson(hiveGraphFile);
- bindings.put("hiveGraphFile", hiveGraphFile);
-
- engine.eval("g.loadGraphSON(hiveGraphFile)", bindings);
- titanGraph.commit();
-
- System.out.println("*******************Graph Dump****************************");
- for (Vertex vertex : titanGraph.getVertices()) {
- System.out.println(GraphHelper.vertexString(vertex));
- }
-
- for (Edge edge : titanGraph.getEdges()) {
- System.out.println(GraphHelper.edgeString(edge));
- }
- System.out.println("*******************Graph Dump****************************");
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- TypeSystem.getInstance().reset();
- }
-
- @Test
- public void testSearchByDSL() throws Exception {
- String dslQuery = "from Department";
-
- String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
-
- JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
- System.out.println("results = " + results);
-
- Object query = results.get("query");
- Assert.assertNotNull(query);
-
- JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
- String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
- Assert.assertEquals(typeName, "Department");
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertEquals(rows.length(), 1);
- }
-
- @Test(expectedExceptions = Throwable.class)
- public void testSearchByDSLBadQuery() throws Exception {
- String dslQuery = "from blah";
-
- discoveryService.searchByDSL(dslQuery);
- Assert.fail();
- }
-
- @Test
- public void testRawSearch1() throws Exception {
- // Query for all Vertices in Graph
- Object r = discoveryService.searchByGremlin("g.V.toList()");
- System.out.println("search result = " + r);
-
- // Query for all Vertices of a Type
- r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Department'}.toList()");
- System.out.println("search result = " + r);
-
- // Property Query: list all Person names
- r = discoveryService
- .searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
- System.out.println("search result = " + r);
- }
-
- @DataProvider(name = "dslQueriesProvider")
- private Object[][] createDSLQueries() {
- return new String[][] {
- {"from DB"},
- {"DB"},
- {"DB where DB.name=\"Reporting\""},
- {"DB DB.name = \"Reporting\""},
- {"DB where DB.name=\"Reporting\" select name, owner"},
- {"DB has name"},
- {"DB, Table"},
- {"DB is JdbcAccess"},
- /*
- {"DB, LoadProcess has name"},
- {"DB as db1, Table where db1.name = \"Reporting\""},
- {"DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
- */
- {"from Table"},
- {"Table"},
- {"Table is Dimension"},
- {"Column where Column isa PII"},
- {"View is Dimension"},
- /*{"Column where Column isa PII select Column.name"},*/
- {"Column select Column.name"},
- {"Column select name"},
- {"Column where Column.name=\"customer_id\""},
- {"from Table select Table.name"},
- {"DB where (name = \"Reporting\")"},
- {"DB where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
- {"DB where DB is JdbcAccess"},
- {"DB where DB has name"},
- {"DB Table"},
- {"DB where DB has name"},
- {"DB as db1 Table where (db1.name = \"Reporting\")"},
- {"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
- /*
- todo: does not work
- {"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"},
- {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
- {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
- {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
- {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
- */
- // trait searches
- {"Dimension"},
- /*{"Fact"}, - todo: does not work*/
- {"JdbcAccess"},
- {"ETL"},
- {"Metric"},
- {"PII"},
- // Lineage
- {"Table LoadProcess outputTable"},
- {"Table loop (LoadProcess outputTable)"},
- {"Table as _loop0 loop (LoadProcess outputTable) withPath"},
- {"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable withPath"},
- {"Table as t, sd, Column as c where t.name=\"sales_fact\" select c.name as colName, c.dataType as colType"},
- {"Table where name='sales_fact', db where name='Reporting'"}
- };
- }
-
- @Test (dataProvider = "dslQueriesProvider")
- public void testSearchByDSLQueries(String dslQuery) throws Exception {
- System.out.println("Executing dslQuery = " + dslQuery);
- String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
-
- JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
- System.out.println("results = " + results);
-
- Object query = results.get("query");
- Assert.assertNotNull(query);
-
- JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
- String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertTrue(rows.length() >= 0); // some queries may not have any results
- System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
- }
-
- @DataProvider(name = "invalidDslQueriesProvider")
- private Object[][] createInvalidDSLQueries() {
- return new String[][] {
- {"from Unknown"},
- {"Unknown"},
- {"Unknown is Blah"},
- };
- }
-
- @Test (dataProvider = "invalidDslQueriesProvider", expectedExceptions = DiscoveryException.class)
- public void testSearchByDSLInvalidQueries(String dslQuery) throws Exception {
- System.out.println("Executing dslQuery = " + dslQuery);
- discoveryService.searchByDSL(dslQuery);
- Assert.fail();
- }
-
- @Test
- public void testSearchForTypeInheritance() throws Exception {
- createTypesWithMultiLevelInheritance();
- createInstances();
-
- String dslQuery = "from D where a = 1";
- String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
-
- JSONObject results = new JSONObject(jsonResults);
- System.out.println("results = " + results);
- }
-
- /*
- * Type Hierarchy is:
- * A(a)
- * B(b) extends A
- * C(c) extends B
- * D(d) extends C
- */
- private void createTypesWithMultiLevelInheritance() throws Exception {
- HierarchicalTypeDefinition A = createClassTypeDef("A", null,
- createRequiredAttrDef("a", DataTypes.INT_TYPE));
-
- HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
- createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
-
- HierarchicalTypeDefinition C = createClassTypeDef("C", ImmutableList.of("B"),
- createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
-
- HierarchicalTypeDefinition D = createClassTypeDef("D", ImmutableList.of("C"),
- createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
-
- TypeSystem.getInstance().defineClassTypes(A, B, C, D);
- }
-
- private void createInstances() throws Exception {
- Referenceable instance = new Referenceable("D");
- instance.set("d", 1);
- instance.set("c", 1);
- instance.set("b", true);
- instance.set("a", 1);
-
- ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, "D");
- ITypedReferenceableInstance typedInstance =
- deptType.convert(instance, Multiplicity.REQUIRED);
-
- repositoryService.createEntity(typedInstance);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/discovery/HiveLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/hadoop/metadata/discovery/HiveLineageServiceTest.java
deleted file mode 100644
index 61526bb..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/discovery/HiveLineageServiceTest.java
+++ /dev/null
@@ -1,590 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.discovery;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.RepositoryMetadataModule;
-import org.apache.hadoop.metadata.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.hadoop.metadata.repository.EntityNotFoundException;
-import org.apache.hadoop.metadata.services.DefaultMetadataService;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-import java.util.List;
-
-/**
- * Unit tests for Hive LineageService.
- */
-@Guice(modules = RepositoryMetadataModule.class)
-public class HiveLineageServiceTest {
-
- @Inject
- private DefaultMetadataService metadataService;
-
- @Inject
- private GraphBackedDiscoveryService discoveryService;
-
- @Inject
- private HiveLineageService hiveLineageService;
-
-// @Inject
-// private GraphProvider<TitanGraph> graphProvider;
-
- @BeforeClass
- public void setUp() throws Exception {
- setUpTypes();
- setupInstances();
-
- // TestUtils.dumpGraph(graphProvider.get());
- }
-
- @DataProvider(name = "dslQueriesProvider")
- private Object[][] createDSLQueries() {
- return new String[][] {
- // joins
- {"hive_table where name=\"sales_fact\", columns"},
- {"hive_table where name=\"sales_fact\", columns select name, dataType, comment"},
- {"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"},
-// {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name, table.name"},
- {"from hive_db"},
- {"hive_db"},
- {"hive_db where hive_db.name=\"Reporting\""},
- {"hive_db hive_db.name = \"Reporting\""},
- {"hive_db where hive_db.name=\"Reporting\" select name, owner"},
- {"hive_db has name"},
-// {"hive_db, hive_table"},
-// {"hive_db, hive_process has name"},
-// {"hive_db as db1, hive_table where db1.name = \"Reporting\""},
-// {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System.currentTimeMillis()},
- {"from hive_table"},
- {"hive_table"},
- {"hive_table is Dimension"},
- {"hive_column where hive_column isa PII"},
-// {"hive_column where hive_column isa PII select hive_column.name"},
- {"hive_column select hive_column.name"},
- {"hive_column select name"},
- {"hive_column where hive_column.name=\"customer_id\""},
- {"from hive_table select hive_table.name"},
- {"hive_db where (name = \"Reporting\")"},
- {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
- {"hive_db where hive_db has name"},
-// {"hive_db hive_table"},
- {"hive_db where hive_db has name"},
-// {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"},
- {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
-// {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"},
-// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
-// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
-// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
-// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
- // trait searches
- {"Dimension"},
- {"Fact"},
- {"ETL"},
- {"Metric"},
- {"PII"},
- };
- }
-
- @Test (dataProvider = "dslQueriesProvider")
- public void testSearchByDSLQueries(String dslQuery) throws Exception {
- System.out.println("Executing dslQuery = " + dslQuery);
- String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
-
- JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
- System.out.println("results = " + results);
-
- Object query = results.get("query");
- Assert.assertNotNull(query);
-
- JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
- String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertTrue(rows.length() >= 0); // some queries may not have any results
- System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
- }
-
- @Test
- public void testGetInputs() throws Exception {
- JSONObject results = new JSONObject(hiveLineageService.getInputs("sales_fact_monthly_mv"));
- Assert.assertNotNull(results);
- System.out.println("inputs = " + results);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertTrue(rows.length() > 0);
-
- final JSONObject row = rows.getJSONObject(0);
- JSONArray paths = row.getJSONArray("path");
- Assert.assertTrue(paths.length() > 0);
- }
-
- @Test (expectedExceptions = IllegalArgumentException.class)
- public void testGetInputsTableNameNull() throws Exception {
- hiveLineageService.getInputs(null);
- Assert.fail();
- }
-
- @Test (expectedExceptions = IllegalArgumentException.class)
- public void testGetInputsTableNameEmpty() throws Exception {
- hiveLineageService.getInputs("");
- Assert.fail();
- }
-
- @Test (expectedExceptions = EntityNotFoundException.class)
- public void testGetInputsBadTableName() throws Exception {
- hiveLineageService.getInputs("blah");
- Assert.fail();
- }
-
- @Test
- public void testGetInputsGraph() throws Exception {
- JSONObject results = new JSONObject(
- hiveLineageService.getInputsGraph("sales_fact_monthly_mv"));
- Assert.assertNotNull(results);
- System.out.println("inputs graph = " + results);
-
- JSONObject values = results.getJSONObject("values");
- Assert.assertNotNull(values);
-
- final JSONObject vertices = values.getJSONObject("vertices");
- Assert.assertEquals(vertices.length(), 4);
-
- final JSONObject edges = values.getJSONObject("edges");
- Assert.assertEquals(edges.length(), 4);
- }
-
- @Test
- public void testGetOutputs() throws Exception {
- JSONObject results = new JSONObject(hiveLineageService.getOutputs("sales_fact"));
- Assert.assertNotNull(results);
- System.out.println("outputs = " + results);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertTrue(rows.length() > 0);
-
- final JSONObject row = rows.getJSONObject(0);
- JSONArray paths = row.getJSONArray("path");
- Assert.assertTrue(paths.length() > 0);
- }
-
- @Test (expectedExceptions = IllegalArgumentException.class)
- public void testGetOututsTableNameNull() throws Exception {
- hiveLineageService.getOutputs(null);
- Assert.fail();
- }
-
- @Test (expectedExceptions = IllegalArgumentException.class)
- public void testGetOutputsTableNameEmpty() throws Exception {
- hiveLineageService.getOutputs("");
- Assert.fail();
- }
-
- @Test (expectedExceptions = EntityNotFoundException.class)
- public void testGetOutputsBadTableName() throws Exception {
- hiveLineageService.getOutputs("blah");
- Assert.fail();
- }
-
- @Test
- public void testGetOutputsGraph() throws Exception {
- JSONObject results = new JSONObject(hiveLineageService.getOutputsGraph("sales_fact"));
- Assert.assertNotNull(results);
- System.out.println("outputs graph = " + results);
-
- JSONObject values = results.getJSONObject("values");
- Assert.assertNotNull(values);
-
- final JSONObject vertices = values.getJSONObject("vertices");
- Assert.assertEquals(vertices.length(), 3);
-
- final JSONObject edges = values.getJSONObject("edges");
- Assert.assertEquals(edges.length(), 4);
- }
-
- @DataProvider(name = "tableNamesProvider")
- private Object[][] tableNames() {
- return new String[][] {
- {"sales_fact", "4"},
- {"time_dim", "3"},
- {"sales_fact_daily_mv", "4"},
- {"sales_fact_monthly_mv", "4"}
- };
- }
-
- @Test (dataProvider = "tableNamesProvider")
- public void testGetSchema(String tableName, String expected) throws Exception {
- JSONObject results = new JSONObject(hiveLineageService.getSchema(tableName));
- Assert.assertNotNull(results);
- System.out.println("columns = " + results);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertEquals(rows.length(), Integer.parseInt(expected));
-
- for (int index = 0; index < rows.length(); index++) {
- final JSONObject row = rows.getJSONObject(index);
- Assert.assertNotNull(row.getString("name"));
- Assert.assertNotNull(row.getString("comment"));
- Assert.assertNotNull(row.getString("dataType"));
- Assert.assertEquals(row.getString("$typeName$"), "hive_column");
- }
- }
-
- @Test (expectedExceptions = IllegalArgumentException.class)
- public void testGetSchemaTableNameNull() throws Exception {
- hiveLineageService.getSchema(null);
- Assert.fail();
- }
-
- @Test (expectedExceptions = IllegalArgumentException.class)
- public void testGetSchemaTableNameEmpty() throws Exception {
- hiveLineageService.getSchema("");
- Assert.fail();
- }
-
- @Test (expectedExceptions = EntityNotFoundException.class)
- public void testGetSchemaBadTableName() throws Exception {
- hiveLineageService.getSchema("blah");
- Assert.fail();
- }
-
- private void setUpTypes() throws Exception {
- TypesDef typesDef = createTypeDefinitions();
- String typesAsJSON = TypesSerialization.toJson(typesDef);
- metadataService.createType(typesAsJSON);
- }
-
- private static final String DATABASE_TYPE = "hive_db";
- private static final String HIVE_TABLE_TYPE = "hive_table";
- private static final String COLUMN_TYPE = "hive_column";
- private static final String HIVE_PROCESS_TYPE = "hive_process";
- private static final String STORAGE_DESC_TYPE = "StorageDesc";
- private static final String VIEW_TYPE = "View";
-
- private TypesDef createTypeDefinitions() {
- HierarchicalTypeDefinition<ClassType> dbClsDef
- = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("description", DataTypes.STRING_TYPE),
- attrDef("locationUri", DataTypes.STRING_TYPE),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> storageDescClsDef =
- TypesUtil.createClassTypeDef(STORAGE_DESC_TYPE, null,
- attrDef("location", DataTypes.STRING_TYPE),
- attrDef("inputFormat", DataTypes.STRING_TYPE),
- attrDef("outputFormat", DataTypes.STRING_TYPE),
- attrDef("compressed", DataTypes.STRING_TYPE,
- Multiplicity.REQUIRED, false, null)
- );
-
- HierarchicalTypeDefinition<ClassType> columnClsDef =
- TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("dataType", DataTypes.STRING_TYPE),
- attrDef("comment", DataTypes.STRING_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> tblClsDef =
- TypesUtil.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE),
- attrDef("lastAccessTime", DataTypes.INT_TYPE),
- attrDef("tableType", DataTypes.STRING_TYPE),
- attrDef("temporary", DataTypes.BOOLEAN_TYPE),
- new AttributeDefinition("db", DATABASE_TYPE,
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("sd", STORAGE_DESC_TYPE,
- Multiplicity.REQUIRED, true, null),
- new AttributeDefinition("columns",
- DataTypes.arrayTypeName(COLUMN_TYPE),
- Multiplicity.COLLECTION, true, null)
- );
-
- HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
- TypesUtil.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
- attrDef("userName", DataTypes.STRING_TYPE),
- attrDef("startTime", DataTypes.INT_TYPE),
- attrDef("endTime", DataTypes.INT_TYPE),
- attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
- );
-
- HierarchicalTypeDefinition<ClassType> viewClsDef =
- TypesUtil.createClassTypeDef(VIEW_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("db", DATABASE_TYPE,
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("inputTables",
- DataTypes.arrayTypeName(HIVE_TABLE_TYPE),
- Multiplicity.COLLECTION, false, null)
- );
-
- HierarchicalTypeDefinition<TraitType> dimTraitDef =
- TypesUtil.createTraitTypeDef("Dimension", null);
-
- HierarchicalTypeDefinition<TraitType> factTraitDef =
- TypesUtil.createTraitTypeDef("Fact", null);
-
- HierarchicalTypeDefinition<TraitType> metricTraitDef =
- TypesUtil.createTraitTypeDef("Metric", null);
-
- HierarchicalTypeDefinition<TraitType> etlTraitDef =
- TypesUtil.createTraitTypeDef("ETL", null);
-
- HierarchicalTypeDefinition<TraitType> piiTraitDef =
- TypesUtil.createTraitTypeDef("PII", null);
-
- HierarchicalTypeDefinition<TraitType> jdbcTraitDef =
- TypesUtil.createTraitTypeDef("JdbcAccess", null);
-
- return TypeUtils.getTypesDef(
- ImmutableList.<EnumTypeDefinition>of(),
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(dimTraitDef, factTraitDef,
- piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
- ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef,
- tblClsDef, loadProcessClsDef, viewClsDef)
- );
- }
-
- AttributeDefinition attrDef(String name, IDataType dT) {
- return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
- return attrDef(name, dT, m, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT,
- Multiplicity m, boolean isComposite, String reverseAttributeName) {
- Preconditions.checkNotNull(name);
- Preconditions.checkNotNull(dT);
- return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
- }
-
- private void setupInstances() throws Exception {
- Id salesDB = database(
- "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
-
- Referenceable sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales",
- "TextInputFormat", "TextOutputFormat", true);
-
- List<Referenceable> salesFactColumns = ImmutableList.of(
- column("time_id", "int", "time id"),
- column("product_id", "int", "product id"),
- column("customer_id", "int", "customer id", "PII"),
- column("sales", "double", "product id", "Metric")
- );
-
- Id salesFact = table("sales_fact", "sales fact table",
- salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
-
- List<Referenceable> timeDimColumns = ImmutableList.of(
- column("time_id", "int", "time id"),
- column("dayOfYear", "int", "day Of Year"),
- column("weekDay", "int", "week Day")
- );
-
- Id timeDim = table("time_dim", "time dimension table",
- salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
-
- Id reportingDB = database("Reporting", "reporting database", "Jane BI",
- "hdfs://host:8000/apps/warehouse/reporting");
-
- Id salesFactDaily = table("sales_fact_daily_mv",
- "sales fact daily materialized view",
- reportingDB, sd, "Joe BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL",
- ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
- "create table as select ", "plan", "id", "graph",
- "ETL");
-
- List<Referenceable> productDimColumns = ImmutableList.of(
- column("product_id", "int", "product id"),
- column("product_name", "string", "product name"),
- column("brand_name", "int", "brand name")
- );
-
- Id productDim = table("product_dim", "product dimension table",
- salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
-
- view("product_dim_view", reportingDB,
- ImmutableList.of(productDim), "Dimension", "JdbcAccess");
-
- List<Referenceable> customerDimColumns = ImmutableList.of(
- column("customer_id", "int", "customer id", "PII"),
- column("name", "string", "customer name", "PII"),
- column("address", "string", "customer address", "PII")
- );
-
- Id customerDim = table("customer_dim", "customer dimension table",
- salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
-
- view("customer_dim_view", reportingDB,
- ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
-
- Id salesFactMonthly = table("sales_fact_monthly_mv",
- "sales fact monthly materialized view",
- reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL",
- ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
- "create table as select ", "plan", "id", "graph",
- "ETL");
- }
-
- Id database(String name, String description,
- String owner, String locationUri,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("locationUri", locationUri);
- referenceable.set("createTime", System.currentTimeMillis());
-
- return createInstance(referenceable);
- }
-
- Referenceable storageDescriptor(String location, String inputFormat,
- String outputFormat,
- boolean compressed) throws Exception {
- Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
- referenceable.set("location", location);
- referenceable.set("inputFormat", inputFormat);
- referenceable.set("outputFormat", outputFormat);
- referenceable.set("compressed", compressed);
-
- return referenceable;
- }
-
- Referenceable column(String name, String dataType, String comment,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("dataType", dataType);
- referenceable.set("comment", comment);
-
- return referenceable;
- }
-
- Id table(String name, String description,
- Id dbId, Referenceable sd,
- String owner, String tableType,
- List<Referenceable> columns,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("tableType", tableType);
- referenceable.set("createTime", System.currentTimeMillis());
- referenceable.set("lastAccessTime", System.currentTimeMillis());
- referenceable.set("retention", System.currentTimeMillis());
-
- referenceable.set("db", dbId);
- referenceable.set("sd", sd);
- referenceable.set("columns", columns);
-
- return createInstance(referenceable);
- }
-
- Id loadProcess(String name, String description, String user,
- List<Id> inputTables,
- List<Id> outputTables,
- String queryText, String queryPlan,
- String queryId, String queryGraph,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("user", user);
- referenceable.set("startTime", System.currentTimeMillis());
- referenceable.set("endTime", System.currentTimeMillis() + 10000);
-
- referenceable.set("inputs", inputTables);
- referenceable.set("outputs", outputTables);
-
- referenceable.set("queryText", queryText);
- referenceable.set("queryPlan", queryPlan);
- referenceable.set("queryId", queryId);
- referenceable.set("queryGraph", queryGraph);
-
- return createInstance(referenceable);
- }
-
- Id view(String name, Id dbId,
- List<Id> inputTables,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("db", dbId);
-
- referenceable.set("inputTables", inputTables);
-
- return createInstance(referenceable);
- }
-
- private Id createInstance(Referenceable referenceable) throws Exception {
- String typeName = referenceable.getTypeName();
- System.out.println("creating instance of type " + typeName);
-
- String entityJSON = InstanceSerialization.toJson(referenceable, true);
- System.out.println("Submitting new entity= " + entityJSON);
- String guid = metadataService.createEntity(entityJSON);
- System.out.println("created instance for type " + typeName + ", guid: " + guid);
-
- // return the reference to created instance with guid
- return new Id(guid, 0, referenceable.getTypeName());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/hadoop/metadata/repository/BaseTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/BaseTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/BaseTest.java
deleted file mode 100755
index f4e9f18..0000000
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/BaseTest.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.memory.MemRepository;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.junit.Before;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.Date;
-import java.util.Map;
-
-public abstract class BaseTest {
-
- public static final String STRUCT_TYPE_1 = "t1";
- public static final String STRUCT_TYPE_2 = "t2";
- public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
- public static final long TEST_DATE_IN_LONG=1418265358440L;
- protected IRepository repo;
-
- public static Struct createStruct() throws MetadataException {
- StructType structType = (StructType) TypeSystem.getInstance()
- .getDataType(StructType.class, STRUCT_TYPE_1);
- Struct s = new Struct(structType.getName());
- s.set("a", 1);
- s.set("b", true);
- s.set("c", (byte) 1);
- s.set("d", (short) 2);
- s.set("e", 1);
- s.set("f", 1);
- s.set("g", 1L);
- s.set("h", 1.0f);
- s.set("i", 1.0);
- s.set("j", BigInteger.valueOf(1L));
- s.set("k", new BigDecimal(1));
- s.set("l", new Date(1418265358440L));
- s.set("m", Lists.<Integer>asList(Integer.valueOf(1), new Integer[]{Integer.valueOf(1)}));
- s.set("n", Lists.<BigDecimal>asList(BigDecimal.valueOf(1.1),
- new BigDecimal[]{BigDecimal.valueOf(1.1)}));
- Map<String, Double> hm = Maps.<String, Double>newHashMap();
- hm.put("a", 1.0);
- hm.put("b", 2.0);
- s.set("o", hm);
- return s;
- }
-
- protected final TypeSystem getTypeSystem() {
- return TypeSystem.getInstance();
- }
-
- protected final IRepository getRepository() {
- return repo;
- }
-
- @Before
- public void setup() throws Exception {
-
- TypeSystem ts = TypeSystem.getInstance();
- ts.reset();
- repo = new MemRepository(ts);
-
- StructType structType = ts.defineStructType(STRUCT_TYPE_1,
- true,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
- TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
- TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
- TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
- TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
- TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
- TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
- TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- TypesUtil.createOptionalAttrDef("o",
- ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
-
- StructType recursiveStructType = ts.defineStructType(STRUCT_TYPE_2,
- true,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
- }
-
- protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs)
- throws MetadataException {
-
- return getTypeSystem().defineTraitTypes(tDefs);
- }
-
- /*
- * Class Hierarchy is:
- * Department(name : String, employees : Array[Person])
- * Person(name : String, department : Department, manager : Manager)
- * Manager(subordinates : Array[Person]) extends Person
- *
- * Persons can have SecurityClearance(level : Int) clearance.
- */
- protected void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
-
- HierarchicalTypeDefinition<ClassType> deptTypeDef =
- TypesUtil.createClassTypeDef("Department", ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, true, "department")
- );
- HierarchicalTypeDefinition<ClassType> personTypeDef =
- TypesUtil.createClassTypeDef("Person", ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department",
- "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager",
- "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- );
- HierarchicalTypeDefinition<ClassType> managerTypeDef =
- TypesUtil.createClassTypeDef("Manager",
- ImmutableList.<String>of("Person"),
- new AttributeDefinition("subordinates",
- String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager")
- );
-
- HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef =
- TypesUtil.createTraitTypeDef("SecurityClearance",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE)
- );
-
- ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
- ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef,
- managerTypeDef));
-
- ImmutableList<HierarchicalType> types = ImmutableList.of(
- ts.getDataType(HierarchicalType.class, "SecurityClearance"),
- ts.getDataType(ClassType.class, "Department"),
- ts.getDataType(ClassType.class, "Person"),
- ts.getDataType(ClassType.class, "Manager")
- );
-
- repo.defineTypes(types);
-
- }
-
- protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
- Referenceable hrDept = new Referenceable("Department");
- Referenceable john = new Referenceable("Person");
- Referenceable jane = new Referenceable("Manager", "SecurityClearance");
-
- hrDept.set("name", "hr");
- john.set("name", "John");
- john.set("department", hrDept);
- jane.set("name", "Jane");
- jane.set("department", hrDept);
-
- john.set("manager", jane);
-
- hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
-
- jane.set("subordinates", ImmutableList.<Referenceable>of(john));
-
- jane.getTrait("SecurityClearance").set("level", 1);
-
- ClassType deptType = ts.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
-
- return hrDept;
- }
-}
[02/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java
deleted file mode 100755
index 1ae733f..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metadata.util.CredentialProviderUtility;
-import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.*;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-/**
- *
- */
-public class CredentialProviderUtilityIT {
-
- private char[] defaultPass = new char[]{'k', 'e', 'y', 'p', 'a', 's', 's'};
-
- @Test
- public void testEnterValidValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- return defaultPass;
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- }
-
- protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry) {
- assertCredentialEntryCorrect(entry, defaultPass);
- }
-
- protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry, char[] password) {
- Assert.assertNotNull(entry);
- Assert.assertEquals(entry.getCredential(), password);
- }
-
- @Test
- public void testEnterEmptyValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
-
- private Random random = new Random();
-
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- List<char[]> responses = new ArrayList<>();
- responses.add(new char[0]);
- responses.add(defaultPass);
-
- int size = responses.size();
- int item = random.nextInt(size);
- return responses.get(item);
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- }
-
- @Test
- public void testEnterMismatchedValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
-
- int i = 0;
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- List<char[]> responses = new ArrayList<>();
- responses.add(defaultPass);
- responses.add(new char[] {'b', 'a', 'd', 'p', 'a', 's', 's'});
- responses.add(defaultPass);
-
- int item = i % 3;
- i++;
- return responses.get(item);
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- }
-
- @Test
- public void testOverwriteValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- return defaultPass;
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- // now attempt to overwrite values
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
-
- int i = 0;
-
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return i++ == 0 ? finalTestPath.toString() : "y";
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- return new char[] {'n', 'e', 'w', 'p', 'a', 's', 's'};
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- char[] newpass = "newpass".toCharArray();
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry, newpass);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry, newpass);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry, newpass);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java
deleted file mode 100644
index 857a42a..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.filters;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdfs.web.URLConnectionFactory;
-import org.apache.hadoop.metadata.security.BaseSecurityTest;
-import org.apache.hadoop.metadata.web.service.EmbeddedServer;
-import org.mortbay.jetty.Server;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import javax.security.auth.Subject;
-import javax.security.auth.callback.*;
-import javax.security.auth.login.LoginContext;
-import javax.security.auth.login.LoginException;
-import java.io.File;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.security.PrivilegedExceptionAction;
-import java.util.Properties;
-
-/**
- *
- */
-public class MetadataAuthenticationKerberosFilterIT extends BaseSecurityTest {
- public static final String TEST_USER_JAAS_SECTION = "TestUser";
- public static final String TESTUSER = "testuser";
- public static final String TESTPASS = "testpass";
-
- private File userKeytabFile;
- private File httpKeytabFile;
-
- class TestEmbeddedServer extends EmbeddedServer {
- public TestEmbeddedServer(int port, String path) throws IOException {
- super(port, path);
- }
-
- Server getServer() {
- return server;
- }
- }
-
- @Test
- public void testKerberosBasedLogin() throws Exception {
- String originalConf = System.getProperty("metadata.conf");
- System.setProperty("metadata.conf", System.getProperty("user.dir"));
-
- setupKDCAndPrincipals();
- TestEmbeddedServer server = null;
-
- try {
- // setup the application.properties file
- generateKerberosTestProperties();
-
- // need to create the web application programmatically in order to control the injection of the test
- // application properties
- server = new TestEmbeddedServer(23000, "webapp/target/metadata-governance");
-
- startEmbeddedServer(server.getServer());
-
- final URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_SYSTEM_CONNECTION_FACTORY;
- // attempt to hit server and get rejected
- URL url = new URL("http://localhost:23000/");
- HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, false);
- connection.setRequestMethod("GET");
- connection.connect();
-
- Assert.assertEquals(connection.getResponseCode(), 401);
-
- // need to populate the ticket cache with a local user, so logging in...
- Subject subject = loginTestUser();
-
- Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- // attempt to hit server and get rejected
- URL url = new URL("http://localhost:23000/");
- HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, true);
- connection.setRequestMethod("GET");
- connection.connect();
-
- Assert.assertEquals(connection.getResponseCode(), 200);
-
- return null;
- }
- });
- } finally {
- server.getServer().stop();
- kdc.stop();
-
- if (originalConf != null) {
- System.setProperty("metadata.conf", originalConf);
- } else {
- System.clearProperty("metadata.conf");
- }
-
- }
-
-
- }
-
- protected Subject loginTestUser() throws LoginException, IOException {
- LoginContext lc = new LoginContext(TEST_USER_JAAS_SECTION, new CallbackHandler() {
-
- @Override
- public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
- for (int i = 0; i < callbacks.length; i++) {
- if (callbacks[i] instanceof PasswordCallback) {
- PasswordCallback passwordCallback = (PasswordCallback) callbacks[i];
- passwordCallback.setPassword(TESTPASS.toCharArray());
- }
- if (callbacks[i] instanceof NameCallback) {
- NameCallback nameCallback = (NameCallback) callbacks[i];
- nameCallback.setName(TESTUSER);
- }
- }
- }
- });
- // attempt authentication
- lc.login();
- return lc.getSubject();
- }
-
- protected void generateKerberosTestProperties() throws IOException, ConfigurationException {
- Properties props = new Properties();
- props.setProperty("metadata.http.authentication.enabled", "true");
- props.setProperty("metadata.http.authentication.type", "kerberos");
- props.setProperty("metadata.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
- props.setProperty("metadata.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
- props.setProperty("metadata.http.authentication.kerberos.name.rules",
- "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
-
- generateTestProperties(props);
- }
-
- public void setupKDCAndPrincipals() throws Exception {
- // set up the KDC
- File kdcWorkDir = startKDC();
-
- userKeytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
- httpKeytabFile = createKeytab(kdc, kdcWorkDir, "HTTP", "spnego.service.keytab");
-
- // create a test user principal
- kdc.createPrincipal(TESTUSER, TESTPASS);
-
- StringBuilder jaas = new StringBuilder(1024);
- jaas.append("TestUser {\n" +
- " com.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true;\n" +
- "};\n");
- jaas.append(createJAASEntry("Client", "dgi", userKeytabFile));
- jaas.append(createJAASEntry("Server", "HTTP", httpKeytabFile));
-
- File jaasFile = new File(kdcWorkDir, "jaas.txt");
- FileUtils.write(jaasFile, jaas.toString());
- bindJVMtoJAASFile(jaasFile);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java
deleted file mode 100644
index f41ad0a..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.filters;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.hadoop.metadata.security.BaseSecurityTest;
-import org.apache.hadoop.metadata.web.service.EmbeddedServer;
-import org.mortbay.jetty.Server;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Properties;
-
-/**
- *
- */
-public class MetadataAuthenticationSimpleFilterIT extends BaseSecurityTest {
-
- class TestEmbeddedServer extends EmbeddedServer {
- public TestEmbeddedServer(int port, String path) throws IOException {
- super(port, path);
- }
-
- Server getServer() {
- return server;
- }
- }
-
- @Test
- public void testSimpleLogin() throws Exception {
- String originalConf = System.getProperty("metadata.conf");
- System.setProperty("metadata.conf", System.getProperty("user.dir"));
- generateSimpleLoginConfiguration();
-
- TestEmbeddedServer server = new TestEmbeddedServer(23001, "webapp/target/metadata-governance");
-
- try {
- startEmbeddedServer(server.getServer());
-
- URL url = new URL("http://localhost:23001");
- HttpURLConnection connection = (HttpURLConnection)url.openConnection();
- connection.setRequestMethod("GET");
- connection.connect();
-
- try {
- Assert.assertEquals(connection.getResponseCode(), 403);
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- url = new URL("http://localhost:23001/?user.name=testuser");
- connection = (HttpURLConnection)url.openConnection();
- connection.setRequestMethod("GET");
- connection.connect();
-
- Assert.assertEquals(connection.getResponseCode(), 200);
- } finally {
- server.getServer().stop();
- if (originalConf != null) {
- System.setProperty("metadata.conf", originalConf);
- } else {
- System.clearProperty("metadata.conf");
- }
- }
-
-
- }
-
- protected void generateSimpleLoginConfiguration() throws IOException, ConfigurationException {
- Properties config = new Properties();
- config.setProperty("metadata.http.authentication.enabled", "true");
- config.setProperty("metadata.http.authentication.type", "simple");
-
- generateTestProperties(config);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java
deleted file mode 100644
index be7171b..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.listeners;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.metadata.security.BaseSecurityTest;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-
-/**
- *
- */
-public class LoginProcessorIT extends BaseSecurityTest {
-
- protected static final String kerberosRule =
- "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT";
-
- @Test
- public void testDefaultSimpleLogin() throws Exception {
- LoginProcessor processor = new LoginProcessor() {
- @Override
- protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
- return new PropertiesConfiguration();
- }
- };
- processor.login();
-
- Assert.assertNotNull(UserGroupInformation.getCurrentUser());
- Assert.assertFalse(UserGroupInformation.isLoginKeytabBased());
- Assert.assertFalse(UserGroupInformation.isSecurityEnabled());
- }
-
- @Test
- public void testKerberosLogin() throws Exception {
- final File keytab = setupKDCAndPrincipals();
-
- LoginProcessor processor = new LoginProcessor() {
- @Override
- protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
- PropertiesConfiguration config = new PropertiesConfiguration();
- config.setProperty("metadata.authentication.method", "kerberos");
- config.setProperty("metadata.authentication.principal", "dgi@EXAMPLE.COM");
- config.setProperty("metadata.authentication.keytab", keytab.getAbsolutePath());
- return config;
- }
-
- @Override
- protected Configuration getHadoopConfiguration() {
- Configuration config = new Configuration(false);
- config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- config.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
- config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, kerberosRule);
-
- return config;
- }
-
- @Override
- protected boolean isHadoopCluster() {
- return true;
- }
- };
- processor.login();
-
- Assert.assertTrue(UserGroupInformation.getLoginUser().getShortUserName().endsWith("dgi"));
- Assert.assertNotNull(UserGroupInformation.getCurrentUser());
- Assert.assertTrue(UserGroupInformation.isLoginKeytabBased());
- Assert.assertTrue(UserGroupInformation.isSecurityEnabled());
-
- kdc.stop();
-
- }
-
- private File setupKDCAndPrincipals() throws Exception {
- // set up the KDC
- File kdcWorkDir = startKDC();
-
- Assert.assertNotNull(kdc.getRealm());
-
- File keytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
-
- return keytabFile;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java
deleted file mode 100755
index ca51cd6..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-
-/**
- * Integration test for Admin jersey resource.
- */
-public class AdminJerseyResourceIT extends BaseResourceIT {
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
- }
-
- @Test
- public void testGetVersion() throws Exception {
- WebResource resource = service
- .path("api/metadata/admin/version");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- PropertiesConfiguration buildConfiguration =
- new PropertiesConfiguration("metadata-buildinfo.properties");
-
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertEquals(response.get("Version"),
- buildConfiguration.getString("build.version"));
- Assert.assertEquals(response.get("Name"),
- buildConfiguration.getString("project.name"));
- Assert.assertEquals(response.get("Description"),
- buildConfiguration.getString("project.description"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java
deleted file mode 100755
index e0efe7d..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriBuilder;
-
-/**
- * Base class for integration tests.
- * Sets up the web resource and has helper methods to create type and entity.
- */
-public abstract class BaseResourceIT {
-
- protected WebResource service;
- protected MetadataServiceClient serviceClient;
- public static String baseUrl = "http://localhost:21000/";
-
- @BeforeClass
- public void setUp() throws Exception {
-
- DefaultClientConfig config = new DefaultClientConfig();
- Client client = Client.create(config);
- client.resource(UriBuilder.fromUri(baseUrl).build());
-
- service = client.resource(UriBuilder.fromUri(baseUrl).build());
- serviceClient = new MetadataServiceClient(baseUrl);
- }
-
- protected void createType(TypesDef typesDef) throws Exception {
- HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0);
- if (serviceClient.getType(sampleType.typeName) == null ) {
- String typesAsJSON = TypesSerialization.toJson(typesDef);
- createType(typesAsJSON);
- }
- }
-
- protected void createType(String typesAsJSON) throws Exception {
- WebResource resource = service
- .path("api/metadata/types");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get("types"));
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- }
-
- protected Id createInstance(Referenceable referenceable) throws Exception {
- String typeName = referenceable.getTypeName();
- System.out.println("creating instance of type " + typeName);
-
- String entityJSON = InstanceSerialization.toJson(referenceable, true);
- System.out.println("Submitting new entity= " + entityJSON);
- JSONObject jsonObject = serviceClient.createEntity(entityJSON);
- String guid = jsonObject.getString(MetadataServiceClient.GUID);
- System.out.println("created instance for type " + typeName + ", guid: " + guid);
-
- // return the reference to created instance with guid
- return new Id(guid, 0, referenceable.getTypeName());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
deleted file mode 100755
index 00cc234..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
+++ /dev/null
@@ -1,679 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.collect.ImmutableList;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.MetadataServiceException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization$;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization$;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * Integration tests for Entity Jersey Resource.
- */
-public class EntityJerseyResourceIT extends BaseResourceIT {
-
- private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class);
-
- private static final String DATABASE_TYPE = "hive_database";
- private static final String DATABASE_NAME = "foo";
- private static final String TABLE_TYPE = "hive_table_type";
- private static final String TABLE_NAME = "bar";
- private static final String TRAITS = "traits";
-
- private Referenceable tableInstance;
- private Id tableId;
- private String traitName;
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
-
- createHiveTypes();
- }
-
- @Test
- public void testSubmitEntity() throws Exception {
- tableInstance = createHiveTableInstance();
- tableId = createInstance(tableInstance);
-
- final String guid = tableId._getId();
- try {
- Assert.assertNotNull(UUID.fromString(guid));
- } catch (IllegalArgumentException e) {
- Assert.fail("Response is not a guid, " + guid);
- }
- }
-
- @DataProvider
- public Object[][] invalidAttrValues() {
- return new Object[][]{
- {null}, {""}, {" "}};
- }
-
- @Test(dataProvider = "invalidAttrValues")
- public void testEntityInvalidValue(String value) throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", randomString());
- databaseInstance.set("description", value);
-
- try {
- createInstance(databaseInstance);
- Assert.fail("Exptected MetadataServiceException");
- } catch(MetadataServiceException e) {
- Assert.assertEquals(e.getStatus(), ClientResponse.Status.BAD_REQUEST);
- }
- }
-
- @Test
- public void testSubmitEntityWithBadDateFormat() throws Exception {
-
- try {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", DATABASE_NAME);
- databaseInstance.set("description", "foo database");
-
- Referenceable tableInstance = new Referenceable(TABLE_TYPE,
- "classification", "pii", "phi", "pci", "sox", "sec", "finance");
- tableInstance.set("name", TABLE_NAME);
- tableInstance.set("description", "bar table");
- tableInstance.set("date", "2014-07-11");
- tableInstance.set("type", "managed");
- tableInstance.set("level", 2);
- tableInstance.set("tableType", 1); // enum
- tableInstance.set("database", databaseInstance);
- tableInstance.set("compressed", false);
-
- Struct traitInstance = (Struct) tableInstance.getTrait("classification");
- traitInstance.set("tag", "foundation_etl");
-
- Struct serde1Instance = new Struct("serdeType");
- serde1Instance.set("name", "serde1");
- serde1Instance.set("serde", "serde1");
- tableInstance.set("serde1", serde1Instance);
-
- Struct serde2Instance = new Struct("serdeType");
- serde2Instance.set("name", "serde2");
- serde2Instance.set("serde", "serde2");
- tableInstance.set("serde2", serde2Instance);
-
- tableId = createInstance(tableInstance);
- Assert.fail("Was expecting an exception here ");
- } catch (MetadataServiceException e) {
- Assert.assertTrue(
- e.getMessage().contains("\"error\":\"Cannot convert value '2014-07-11' to datatype date\""));
- }
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testAddProperty() throws Exception {
- final String guid = tableId._getId();
- //add property
- String description = "bar table - new desc";
- ClientResponse clientResponse = addProperty(guid, "description", description);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String entityRef = getEntityDefinition(getEntityDefinition(guid));
- Assert.assertNotNull(entityRef);
-
- tableInstance.set("description", description);
-
- //invalid property for the type
- clientResponse = addProperty(guid, "invalid_property", "bar table");
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
-
- //non-string property, update
- clientResponse = addProperty(guid, "level", "4");
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- entityRef = getEntityDefinition(getEntityDefinition(guid));
- Assert.assertNotNull(entityRef);
-
- tableInstance.set("level", 4);
- }
-
- @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
- public void testAddNullProperty() throws Exception {
- final String guid = tableId._getId();
- //add property
- addProperty(guid, null, "foo bar");
- Assert.fail();
- }
-
- @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
- public void testAddNullPropertyValue() throws Exception {
- final String guid = tableId._getId();
- //add property
- addProperty(guid, "description", null);
- Assert.fail();
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testAddReferenceProperty() throws Exception {
- //Create new db instance
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", "newdb");
- databaseInstance.set("description", "new database");
-
- Id dbInstance = createInstance(databaseInstance);
- String dbId = dbInstance._getId();
-
- //Add reference property
- final String guid = tableId._getId();
- ClientResponse clientResponse = addProperty(guid, "database", dbId);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testGetEntityDefinition() throws Exception {
- final String guid = tableId._getId();
- ClientResponse clientResponse = getEntityDefinition(guid);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final String definition = response.getString(MetadataServiceClient.DEFINITION);
- Assert.assertNotNull(definition);
- LOG.debug("tableInstanceAfterGet = " + definition);
- InstanceSerialization.fromJsonReferenceable(definition, true);
- }
-
- private ClientResponse addProperty(String guid, String property, String value) {
- WebResource resource = service
- .path("api/metadata/entities")
- .path(guid);
-
- return resource.queryParam("property", property).queryParam("value", value)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.PUT, ClientResponse.class);
- }
-
- private ClientResponse getEntityDefinition(String guid) {
- WebResource resource = service
- .path("api/metadata/entities")
- .path(guid);
- return resource.accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- }
-
- private String getEntityDefinition(ClientResponse clientResponse) throws Exception {
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
- JSONObject response = new JSONObject(clientResponse.getEntity(String.class));
- final String definition = response.getString(MetadataServiceClient.DEFINITION);
- Assert.assertNotNull(definition);
-
- return definition;
- }
-
- @Test
- public void testGetInvalidEntityDefinition() throws Exception {
- WebResource resource = service
- .path("api/metadata/entities")
- .path("blah");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
- Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testGetEntityList() throws Exception {
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .queryParam("type", TABLE_TYPE)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(list);
- Assert.assertEquals(list.length(), 1);
- }
-
- @Test
- public void testGetEntityListForBadEntityType() throws Exception {
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .queryParam("type", "blah")
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
- Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
- }
-
-
- @Test
- public void testGetEntityListForNoInstances() throws Exception {
- addNewType();
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .queryParam("type", "test")
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertEquals(list.length(), 0);
- }
-
- private void addNewType() throws Exception {
- HierarchicalTypeDefinition<ClassType> testTypeDefinition =
- TypesUtil.createClassTypeDef("test", ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
- String typesAsJSON = TypesSerialization.toJson(testTypeDefinition);
- createType(typesAsJSON);
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testGetTraitNames() throws Exception {
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get("GUID"));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertEquals(list.length(), 7);
- }
-
- @Test(dependsOnMethods = "testGetTraitNames")
- public void testAddTrait() throws Exception {
- traitName = "PII_Trait" + randomString();
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
- String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
- LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
- createType(traitDefinitionAsJSON);
-
- Struct traitInstance = new Struct(traitName);
- String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get(MetadataServiceClient.GUID));
- }
-
- @Test(dependsOnMethods = "testAddTrait")
- public void testAddExistingTrait() throws Exception {
- final String traitName = "PII_Trait" + randomString();
-
- Struct traitInstance = new Struct(traitName);
- String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
- }
-
- @Test(dependsOnMethods = "testGetTraitNames")
- public void testAddTraitWithAttribute() throws Exception {
- final String traitName = "PII_Trait" + randomString();
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
- String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
- LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
- createType(traitDefinitionAsJSON);
-
- Struct traitInstance = new Struct(traitName);
- traitInstance.set("type", "SSN");
- String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get(MetadataServiceClient.GUID));
-
- // verify the response
- clientResponse = getEntityDefinition(guid);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
- responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final String definition = response.getString(MetadataServiceClient.DEFINITION);
- Assert.assertNotNull(definition);
- Referenceable entityRef = InstanceSerialization.fromJsonReferenceable(definition, true);
- IStruct traitRef = entityRef.getTrait(traitName);
- String type = (String) traitRef.get("type");
- Assert.assertEquals(type, "SSN");
- }
-
- @Test
- public void testAddTraitWithNoRegistration() throws Exception {
- final String traitName = "PII_Trait" + randomString();
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
- String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
- LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
-
- Struct traitInstance = new Struct(traitName);
- String traitInstanceAsJSON = InstanceSerialization$.MODULE$.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path("random")
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
- }
-
- @Test(dependsOnMethods = "testAddTrait")
- public void testDeleteTrait() throws Exception {
- final String guid = tableId._getId();
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .path(traitName)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.DELETE, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get("GUID"));
- Assert.assertNotNull(response.get("traitName"));
- }
-
- @Test
- public void testDeleteTraitNonExistent() throws Exception {
- final String traitName = "blah_trait";
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path("random")
- .path(TRAITS)
- .path(traitName)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.DELETE, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
- Assert.assertEquals(response.getString(MetadataServiceClient.ERROR),
- "trait=" + traitName + " should be defined in type system before it can be deleted");
- Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
- }
-
- private String random() {
- return RandomStringUtils.random(10);
- }
-
- private String randomString() {
- return RandomStringUtils.randomAlphanumeric(10);
- }
-
- @Test
- public void testUTF8() throws Exception {
- String classType = random();
- String attrName = random();
- String attrValue = random();
-
- HierarchicalTypeDefinition<ClassType> classTypeDefinition =
- TypesUtil.createClassTypeDef(classType, ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE));
- TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
- ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.of(classTypeDefinition));
- createType(typesDef);
-
- Referenceable instance = new Referenceable(classType);
- instance.set(attrName, attrValue);
- Id guid = createInstance(instance);
-
- ClientResponse response = getEntityDefinition(guid._getId());
- String definition = getEntityDefinition(response);
- Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
- Assert.assertEquals(getReferenceable.get(attrName), attrValue);
- }
-
- private void createHiveTypes() throws Exception {
- HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
- TypesUtil.createClassTypeDef(DATABASE_TYPE,
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
- StructTypeDefinition structTypeDefinition =
- new StructTypeDefinition("serdeType",
- new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
- });
-
- EnumValue values[] = {
- new EnumValue("MANAGED", 1),
- new EnumValue("EXTERNAL", 2),
- };
-
- EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
-
- HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
- TypesUtil.createClassTypeDef(TABLE_TYPE,
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("date", DataTypes.DATE_TYPE),
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE),
- new AttributeDefinition("tableType", "tableType",
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("serde1",
- "serdeType", Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("serde2",
- "serdeType", Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("database",
- DATABASE_TYPE, Multiplicity.REQUIRED, true, null),
- new AttributeDefinition("compressed",
- DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, true, null));
-
- HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
- TypesUtil.createTraitTypeDef("classification",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef("pii", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> phiTrait =
- TypesUtil.createTraitTypeDef("phi", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> pciTrait =
- TypesUtil.createTraitTypeDef("pci", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> soxTrait =
- TypesUtil.createTraitTypeDef("sox", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> secTrait =
- TypesUtil.createTraitTypeDef("sec", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> financeTrait =
- TypesUtil.createTraitTypeDef("finance", ImmutableList.<String>of());
-
- TypesDef typesDef = TypeUtils.getTypesDef(
- ImmutableList.of(enumTypeDefinition),
- ImmutableList.of(structTypeDefinition),
- ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
- soxTrait, secTrait, financeTrait),
- ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
- createType(typesDef);
- }
-
- private Referenceable createHiveTableInstance() throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", DATABASE_NAME);
- databaseInstance.set("description", "foo database");
-
- Referenceable tableInstance = new Referenceable(TABLE_TYPE,
- "classification", "pii", "phi", "pci", "sox", "sec", "finance");
- tableInstance.set("name", TABLE_NAME);
- tableInstance.set("description", "bar table");
- tableInstance.set("date", "2014-07-11T08:00:00.000Z");
- tableInstance.set("type", "managed");
- tableInstance.set("level", 2);
- tableInstance.set("tableType", 1); // enum
- tableInstance.set("database", databaseInstance);
- tableInstance.set("compressed", false);
-
- Struct traitInstance = (Struct) tableInstance.getTrait("classification");
- traitInstance.set("tag", "foundation_etl");
-
- Struct serde1Instance = new Struct("serdeType");
- serde1Instance.set("name", "serde1");
- serde1Instance.set("serde", "serde1");
- tableInstance.set("serde1", serde1Instance);
-
- Struct serde2Instance = new Struct("serdeType");
- serde2Instance.set("name", "serde2");
- serde2Instance.set("serde", "serde2");
- tableInstance.set("serde2", serde2Instance);
-
- List<String> traits = tableInstance.getTraits();
- Assert.assertEquals(traits.size(), 7);
-
- return tableInstance;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java
deleted file mode 100644
index a40f7ce..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import java.util.List;
-
-/**
- * Hive Lineage Integration Tests.
- */
-public class HiveLineageJerseyResourceIT extends BaseResourceIT {
-
- private static final String BASE_URI = "api/metadata/lineage/hive/table/";
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
-
- setUpTypes();
- setupInstances();
- }
-
- @Test
- public void testInputsGraph() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("sales_fact_monthly_mv")
- .path("inputs")
- .path("graph");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- System.out.println("inputs graph = " + responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(results);
-
- JSONObject values = results.getJSONObject("values");
- Assert.assertNotNull(values);
-
- final JSONObject vertices = values.getJSONObject("vertices");
- Assert.assertEquals(vertices.length(), 4);
-
- final JSONObject edges = values.getJSONObject("edges");
- Assert.assertEquals(edges.length(), 4);
- }
-
- @Test
- public void testOutputsGraph() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("sales_fact")
- .path("outputs")
- .path("graph");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- System.out.println("outputs graph= " + responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(results);
-
- JSONObject values = results.getJSONObject("values");
- Assert.assertNotNull(values);
-
- final JSONObject vertices = values.getJSONObject("vertices");
- Assert.assertEquals(vertices.length(), 3);
-
- final JSONObject edges = values.getJSONObject("edges");
- Assert.assertEquals(edges.length(), 4);
- }
-
- @Test
- public void testSchema() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("sales_fact")
- .path("schema");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- System.out.println("schema = " + responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(results);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertEquals(rows.length(), 4);
-
- for (int index = 0; index < rows.length(); index++) {
- final JSONObject row = rows.getJSONObject(index);
- Assert.assertNotNull(row.getString("name"));
- Assert.assertNotNull(row.getString("comment"));
- Assert.assertNotNull(row.getString("dataType"));
- Assert.assertEquals(row.getString("$typeName$"), "hive_column");
- }
- }
-
- @Test
- public void testSchemaForEmptyTable() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("")
- .path("schema");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(),
- Response.Status.NOT_FOUND.getStatusCode());
- }
-
- @Test
- public void testSchemaForInvalidTable() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("blah")
- .path("schema");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(),
- Response.Status.NOT_FOUND.getStatusCode());
- }
-
- private void setUpTypes() throws Exception {
- TypesDef typesDef = createTypeDefinitions();
- createType(typesDef);
- }
-
- private static final String DATABASE_TYPE = "hive_db";
- private static final String HIVE_TABLE_TYPE = "hive_table";
- private static final String COLUMN_TYPE = "hive_column";
- private static final String HIVE_PROCESS_TYPE = "hive_process";
-
- private TypesDef createTypeDefinitions() {
- HierarchicalTypeDefinition<ClassType> dbClsDef
- = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("description", DataTypes.STRING_TYPE),
- attrDef("locationUri", DataTypes.STRING_TYPE),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> columnClsDef =
- TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("dataType", DataTypes.STRING_TYPE),
- attrDef("comment", DataTypes.STRING_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> tblClsDef =
- TypesUtil.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE),
- attrDef("lastAccessTime", DataTypes.INT_TYPE),
- attrDef("tableType", DataTypes.STRING_TYPE),
- attrDef("temporary", DataTypes.BOOLEAN_TYPE),
- new AttributeDefinition("db", DATABASE_TYPE,
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("columns",
- DataTypes.arrayTypeName(COLUMN_TYPE),
- Multiplicity.COLLECTION, true, null)
- );
-
- HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
- TypesUtil.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
- attrDef("userName", DataTypes.STRING_TYPE),
- attrDef("startTime", DataTypes.INT_TYPE),
- attrDef("endTime", DataTypes.INT_TYPE),
- attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
- );
-
- HierarchicalTypeDefinition<TraitType> dimTraitDef =
- TypesUtil.createTraitTypeDef("Dimension", null);
-
- HierarchicalTypeDefinition<TraitType> factTraitDef =
- TypesUtil.createTraitTypeDef("Fact", null);
-
- HierarchicalTypeDefinition<TraitType> metricTraitDef =
- TypesUtil.createTraitTypeDef("Metric", null);
-
- HierarchicalTypeDefinition<TraitType> etlTraitDef =
- TypesUtil.createTraitTypeDef("ETL", null);
-
-
- HierarchicalTypeDefinition<TraitType> piiTraitDef =
- TypesUtil.createTraitTypeDef("PII", null);
-
- return TypeUtils.getTypesDef(
- ImmutableList.<EnumTypeDefinition>of(),
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef, piiTraitDef),
- ImmutableList.of(dbClsDef, columnClsDef, tblClsDef, loadProcessClsDef)
- );
- }
-
- AttributeDefinition attrDef(String name, IDataType dT) {
- return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
- return attrDef(name, dT, m, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT,
- Multiplicity m, boolean isComposite, String reverseAttributeName) {
- Preconditions.checkNotNull(name);
- Preconditions.checkNotNull(dT);
- return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
- }
-
- private void setupInstances() throws Exception {
- Id salesDB = database(
- "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
-
- List<Referenceable> salesFactColumns = ImmutableList.of(
- column("time_id", "int", "time id"),
- column("product_id", "int", "product id"),
- column("customer_id", "int", "customer id", "PII"),
- column("sales", "double", "product id", "Metric")
- );
-
- Id salesFact = table("sales_fact", "sales fact table",
- salesDB, "Joe", "Managed", salesFactColumns, "Fact");
-
- List<Referenceable> timeDimColumns = ImmutableList.of(
- column("time_id", "int", "time id"),
- column("dayOfYear", "int", "day Of Year"),
- column("weekDay", "int", "week Day")
- );
-
- Id timeDim = table("time_dim", "time dimension table",
- salesDB, "John Doe", "External", timeDimColumns, "Dimension");
-
- Id reportingDB = database("Reporting", "reporting database", "Jane BI",
- "hdfs://host:8000/apps/warehouse/reporting");
-
- Id salesFactDaily = table("sales_fact_daily_mv",
- "sales fact daily materialized view",
- reportingDB, "Joe BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesDaily", "John ETL",
- ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
- "create table as select ", "plan", "id", "graph",
- "ETL");
-
- Id salesFactMonthly = table("sales_fact_monthly_mv",
- "sales fact monthly materialized view",
- reportingDB, "Jane BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesMonthly", "John ETL",
- ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
- "create table as select ", "plan", "id", "graph",
- "ETL");
- }
-
- Id database(String name, String description,
- String owner, String locationUri,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("locationUri", locationUri);
- referenceable.set("createTime", System.currentTimeMillis());
-
- return createInstance(referenceable);
- }
-
- Referenceable column(String name, String dataType, String comment,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("dataType", dataType);
- referenceable.set("comment", comment);
-
- return referenceable;
- }
-
- Id table(String name, String description, Id dbId,
- String owner, String tableType,
- List<Referenceable> columns,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("tableType", tableType);
- referenceable.set("createTime", System.currentTimeMillis());
- referenceable.set("lastAccessTime", System.currentTimeMillis());
- referenceable.set("retention", System.currentTimeMillis());
-
- referenceable.set("db", dbId);
- referenceable.set("columns", columns);
-
- return createInstance(referenceable);
- }
-
- Id loadProcess(String name, String user,
- List<Id> inputTables,
- List<Id> outputTables,
- String queryText, String queryPlan,
- String queryId, String queryGraph,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("user", user);
- referenceable.set("startTime", System.currentTimeMillis());
- referenceable.set("endTime", System.currentTimeMillis() + 10000);
-
- referenceable.set("inputs", inputTables);
- referenceable.set("outputs", outputTables);
-
- referenceable.set("queryText", queryText);
- referenceable.set("queryPlan", queryPlan);
- referenceable.set("queryId", queryId);
- referenceable.set("queryGraph", queryGraph);
-
- return createInstance(referenceable);
- }
-}
[21/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/atlas_config.py
----------------------------------------------------------------------
diff --git a/src/bin/atlas_config.py b/src/bin/atlas_config.py
new file mode 100755
index 0000000..f8ffe45
--- /dev/null
+++ b/src/bin/atlas_config.py
@@ -0,0 +1,285 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import getpass
+
+import os
+import platform
+import subprocess
+from threading import Thread
+import sys
+import time
+import errno
+
+LIB = "lib"
+CONF = "conf"
+LOG="logs"
+WEBAPP="server" + os.sep + "webapp"
+DATA="data"
+ENV_KEYS = ["JAVA_HOME", "METADATA_OPTS", "METADATA_LOG_DIR", "METADATA_PID_DIR", "METADATA_CONF", "METADATACPPATH", "METADATA_DATA_DIR", "METADATA_HOME_DIR", "METADATA_EXPANDED_WEBAPP_DIR"]
+METADATA_CONF = "METADATA_CONF"
+METADATA_LOG = "METADATA_LOG_DIR"
+METADATA_PID = "METADATA_PID_DIR"
+METADATA_WEBAPP = "METADATA_EXPANDED_WEBAPP_DIR"
+METADATA_OPTS = "METADATA_OPTS"
+METADATA_DATA = "METADATA_DATA_DIR"
+METADATA_HOME = "METADATA_HOME_DIR"
+IS_WINDOWS = platform.system() == "Windows"
+ON_POSIX = 'posix' in sys.builtin_module_names
+DEBUG = False
+
+def scriptDir():
+ """
+ get the script path
+ """
+ return os.path.dirname(os.path.realpath(__file__))
+
+def metadataDir():
+ home = os.path.dirname(scriptDir())
+ return os.environ.get(METADATA_HOME, home)
+
+def libDir(dir) :
+ return os.path.join(dir, LIB)
+
+def confDir(dir):
+ localconf = os.path.join(dir, CONF)
+ return os.environ.get(METADATA_CONF, localconf)
+
+def logDir(dir):
+ localLog = os.path.join(dir, LOG)
+ return os.environ.get(METADATA_LOG, localLog)
+
+def pidFile(dir):
+ localPid = os.path.join(dir, LOG)
+ return os.path.join(os.environ.get(METADATA_PID, localPid), 'metadata.pid')
+
+def dataDir(dir):
+ data = os.path.join(dir, DATA)
+ return os.environ.get(METADATA_DATA, data)
+
+def webAppDir(dir):
+ webapp = os.path.join(dir, WEBAPP)
+ return os.environ.get(METADATA_WEBAPP, webapp)
+
+def expandWebApp(dir):
+ webappDir = webAppDir(dir)
+ webAppMetadataDir = os.path.join(webappDir, "metadata")
+ d = os.sep
+ if not os.path.exists(os.path.join(webAppMetadataDir, "WEB-INF")):
+ try:
+ os.makedirs(webAppMetadataDir)
+ except OSError, e:
+ if e.errno != errno.EEXIST:
+ raise e
+ pass
+ os.chdir(webAppMetadataDir)
+ jar(os.path.join(metadataDir(), "server", "webapp", "atlas.war"))
+
+def dirMustExist(dirname):
+ if not os.path.exists(dirname):
+ os.mkdir(dirname)
+ return dirname
+
+def executeEnvSh(confDir):
+ envscript = '%s/metadata-env.sh' % confDir
+ if not IS_WINDOWS and os.path.exists(envscript):
+ envCmd = 'source %s && env' % envscript
+ command = ['bash', '-c', envCmd]
+
+ proc = subprocess.Popen(command, stdout = subprocess.PIPE)
+
+ for line in proc.stdout:
+ (key, _, value) = line.strip().partition("=")
+ if key in ENV_KEYS:
+ os.environ[key] = value
+
+ proc.communicate()
+
+def java(classname, args, classpath, jvm_opts_list, logdir=None):
+ java_home = os.environ.get("JAVA_HOME", None)
+ if java_home:
+ prg = os.path.join(java_home, "bin", "java")
+ else:
+ prg = which("java")
+
+ commandline = [prg]
+ commandline.extend(jvm_opts_list)
+ commandline.append("-classpath")
+ commandline.append(classpath)
+ commandline.append(classname)
+ commandline.extend(args)
+ return runProcess(commandline, logdir)
+
+def jar(path):
+ java_home = os.environ.get("JAVA_HOME", None)
+ if java_home:
+ prg = os.path.join(java_home, "bin", "jar")
+ else:
+ prg = which("jar")
+
+ commandline = [prg]
+ commandline.append("-xf")
+ commandline.append(path)
+ process = runProcess(commandline)
+ process.wait()
+
+def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+def which(program):
+
+ fpath, fname = os.path.split(program)
+ if fpath:
+ if is_exe(program):
+ return program
+ else:
+ for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ exe_file = os.path.join(path, program)
+ if is_exe(exe_file):
+ return exe_file
+
+ return None
+
+def runProcess(commandline, logdir=None):
+ """
+ Run a process
+ :param commandline: command line
+ :return:the return code
+ """
+ global finished
+ debug ("Executing : %s" % commandline)
+ timestr = time.strftime("metadata.%Y%m%d-%H%M%S")
+ stdoutFile = None
+ stderrFile = None
+ if logdir:
+ stdoutFile = open(os.path.join(logdir, timestr + ".out"), "w")
+ stderrFile = open(os.path.join(logdir,timestr + ".err"), "w")
+ return subprocess.Popen(commandline, stdout=stdoutFile, stderr=stderrFile)
+
+def print_output(name, src, toStdErr):
+ """
+ Relay the output stream to stdout line by line
+ :param name:
+ :param src: source stream
+ :param toStdErr: flag set if stderr is to be the dest
+ :return:
+ """
+
+ global needPassword
+ debug ("starting printer for %s" % name )
+ line = ""
+ while not finished:
+ (line, done) = read(src, line)
+ if done:
+ out(toStdErr, line + "\n")
+ flush(toStdErr)
+ if line.find("Enter password for") >= 0:
+ needPassword = True
+ line = ""
+ out(toStdErr, line)
+ # closedown: read remainder of stream
+ c = src.read(1)
+ while c!="" :
+ c = c.decode('utf-8')
+ out(toStdErr, c)
+ if c == "\n":
+ flush(toStdErr)
+ c = src.read(1)
+ flush(toStdErr)
+ src.close()
+
+def read_input(name, exe):
+ """
+ Read input from stdin and send to process
+ :param name:
+ :param process: process to send input to
+ :return:
+ """
+ global needPassword
+ debug ("starting reader for %s" % name )
+ while not finished:
+ if needPassword:
+ needPassword = False
+ if sys.stdin.isatty():
+ cred = getpass.getpass()
+ else:
+ cred = sys.stdin.readline().rstrip()
+ exe.stdin.write(cred + "\n")
+
+def debug(text):
+ if DEBUG: print '[DEBUG] ' + text
+
+
+def error(text):
+ print '[ERROR] ' + text
+ sys.stdout.flush()
+
+def info(text):
+ print text
+ sys.stdout.flush()
+
+
+def out(toStdErr, text) :
+ """
+ Write to one of the system output channels.
+ This action does not add newlines. If you want that: write them yourself
+ :param toStdErr: flag set if stderr is to be the dest
+ :param text: text to write.
+ :return:
+ """
+ if toStdErr:
+ sys.stderr.write(text)
+ else:
+ sys.stdout.write(text)
+
+def flush(toStdErr) :
+ """
+ Flush the output stream
+ :param toStdErr: flag set if stderr is to be the dest
+ :return:
+ """
+ if toStdErr:
+ sys.stderr.flush()
+ else:
+ sys.stdout.flush()
+
+def read(pipe, line):
+ """
+ read a char, append to the listing if there is a char that is not \n
+ :param pipe: pipe to read from
+ :param line: line being built up
+ :return: (the potentially updated line, flag indicating newline reached)
+ """
+
+ c = pipe.read(1)
+ if c != "":
+ o = c.decode('utf-8')
+ if o != '\n':
+ line += o
+ return line, False
+ else:
+ return line, True
+ else:
+ return line, False
+
+def writePid(metadata_pid_file, process):
+ f = open(metadata_pid_file, 'w')
+ f.write(str(process.pid))
+ f.close()
+
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/atlas_start.py
----------------------------------------------------------------------
diff --git a/src/bin/atlas_start.py b/src/bin/atlas_start.py
new file mode 100755
index 0000000..74b242b
--- /dev/null
+++ b/src/bin/atlas_start.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import sys
+import traceback
+
+import atlas_config as mc
+
+METADATA_LOG_OPTS="-Dmetadata.log.dir=%s -Dmetadata.log.file=application.log"
+METADATA_COMMAND_OPTS="-Dmetadata.home=%s"
+METADATA_CONFIG_OPTS="-Dmetadata.conf=%s"
+DEFAULT_JVM_OPTS="-Xmx1024m"
+
+def main():
+
+ metadata_home = mc.metadataDir()
+ confdir = mc.dirMustExist(mc.confDir(metadata_home))
+ mc.executeEnvSh(confdir)
+ logdir = mc.dirMustExist(mc.logDir(metadata_home))
+
+ #create sys property for conf dirs
+ jvm_opts_list = (METADATA_LOG_OPTS % logdir).split()
+
+ cmd_opts = (METADATA_COMMAND_OPTS % metadata_home)
+ jvm_opts_list.extend(cmd_opts.split())
+
+ config_opts = (METADATA_CONFIG_OPTS % confdir)
+ jvm_opts_list.extend(config_opts.split())
+
+ default_jvm_opts = DEFAULT_JVM_OPTS
+ metadata_jvm_opts = os.environ.get(mc.METADATA_OPTS, default_jvm_opts)
+ jvm_opts_list.extend(metadata_jvm_opts.split())
+
+ #expand web app dir
+ web_app_dir = mc.webAppDir(metadata_home)
+ mc.expandWebApp(metadata_home)
+
+ p = os.pathsep
+ metadata_classpath = confdir + p \
+ + os.path.join(web_app_dir, "metadata", "WEB-INF", "classes" ) + p \
+ + os.path.join(web_app_dir, "metadata", "WEB-INF", "lib", "*" ) + p \
+ + os.path.join(metadata_home, "libext", "*")
+
+ metadata_pid_file = mc.pidFile(metadata_home)
+
+ if os.path.isfile(metadata_pid_file):
+ print "%s already exists, exiting" % metadata_pid_file
+ sys.exit()
+
+ args = ["-app", os.path.join(web_app_dir, "metadata")]
+ args.extend(sys.argv[1:])
+
+ process = mc.java("org.apache.atlas.Main", args, metadata_classpath, jvm_opts_list, logdir)
+ mc.writePid(metadata_pid_file, process)
+
+ print "Apache Atlas Server started!!!\n"
+
+if __name__ == '__main__':
+ try:
+ returncode = main()
+ except Exception as e:
+ print "Exception: %s " % str(e)
+ print traceback.format_exc()
+ returncode = -1
+
+ sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/atlas_stop.py
----------------------------------------------------------------------
diff --git a/src/bin/atlas_stop.py b/src/bin/atlas_stop.py
new file mode 100755
index 0000000..a157427
--- /dev/null
+++ b/src/bin/atlas_stop.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from signal import SIGTERM
+import sys
+import traceback
+
+import atlas_config as mc
+
+def main():
+
+ metadata_home = mc.metadataDir()
+ confdir = mc.dirMustExist(mc.confDir(metadata_home))
+ mc.executeEnvSh(confdir)
+ piddir = mc.dirMustExist(mc.logDir(metadata_home))
+
+ metadata_pid_file = mc.pidFile(metadata_home)
+
+ try:
+ pf = file(metadata_pid_file, 'r')
+ pid = int(pf.read().strip())
+ pf.close()
+ except:
+ pid = None
+
+ if not pid:
+ sys.stderr.write("No process ID file found. Server not running?\n")
+ return
+
+ os.kill(pid, SIGTERM)
+
+ # assuming kill worked since process check on windows is more involved...
+ if os.path.exists(metadata_pid_file):
+ os.remove(metadata_pid_file)
+
+if __name__ == '__main__':
+ try:
+ returncode = main()
+ except Exception as e:
+ print "Exception: %s " % str(e)
+ print traceback.format_exc()
+ returncode = -1
+
+ sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/metadata_config.py
----------------------------------------------------------------------
diff --git a/src/bin/metadata_config.py b/src/bin/metadata_config.py
deleted file mode 100755
index 8c688ab..0000000
--- a/src/bin/metadata_config.py
+++ /dev/null
@@ -1,285 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import getpass
-
-import os
-import platform
-import subprocess
-from threading import Thread
-import sys
-import time
-import errno
-
-LIB = "lib"
-CONF = "conf"
-LOG="logs"
-WEBAPP="server" + os.sep + "webapp"
-DATA="data"
-ENV_KEYS = ["JAVA_HOME", "METADATA_OPTS", "METADATA_LOG_DIR", "METADATA_PID_DIR", "METADATA_CONF", "METADATACPPATH", "METADATA_DATA_DIR", "METADATA_HOME_DIR", "METADATA_EXPANDED_WEBAPP_DIR"]
-METADATA_CONF = "METADATA_CONF"
-METADATA_LOG = "METADATA_LOG_DIR"
-METADATA_PID = "METADATA_PID_DIR"
-METADATA_WEBAPP = "METADATA_EXPANDED_WEBAPP_DIR"
-METADATA_OPTS = "METADATA_OPTS"
-METADATA_DATA = "METADATA_DATA_DIR"
-METADATA_HOME = "METADATA_HOME_DIR"
-IS_WINDOWS = platform.system() == "Windows"
-ON_POSIX = 'posix' in sys.builtin_module_names
-DEBUG = False
-
-def scriptDir():
- """
- get the script path
- """
- return os.path.dirname(os.path.realpath(__file__))
-
-def metadataDir():
- home = os.path.dirname(scriptDir())
- return os.environ.get(METADATA_HOME, home)
-
-def libDir(dir) :
- return os.path.join(dir, LIB)
-
-def confDir(dir):
- localconf = os.path.join(dir, CONF)
- return os.environ.get(METADATA_CONF, localconf)
-
-def logDir(dir):
- localLog = os.path.join(dir, LOG)
- return os.environ.get(METADATA_LOG, localLog)
-
-def pidFile(dir):
- localPid = os.path.join(dir, LOG)
- return os.path.join(os.environ.get(METADATA_PID, localPid), 'metadata.pid')
-
-def dataDir(dir):
- data = os.path.join(dir, DATA)
- return os.environ.get(METADATA_DATA, data)
-
-def webAppDir(dir):
- webapp = os.path.join(dir, WEBAPP)
- return os.environ.get(METADATA_WEBAPP, webapp)
-
-def expandWebApp(dir):
- webappDir = webAppDir(dir)
- webAppMetadataDir = os.path.join(webappDir, "metadata")
- d = os.sep
- if not os.path.exists(os.path.join(webAppMetadataDir, "WEB-INF")):
- try:
- os.makedirs(webAppMetadataDir)
- except OSError, e:
- if e.errno != errno.EEXIST:
- raise e
- pass
- os.chdir(webAppMetadataDir)
- jar(os.path.join(metadataDir(), "server", "webapp", "metadata.war"))
-
-def dirMustExist(dirname):
- if not os.path.exists(dirname):
- os.mkdir(dirname)
- return dirname
-
-def executeEnvSh(confDir):
- envscript = '%s/metadata-env.sh' % confDir
- if not IS_WINDOWS and os.path.exists(envscript):
- envCmd = 'source %s && env' % envscript
- command = ['bash', '-c', envCmd]
-
- proc = subprocess.Popen(command, stdout = subprocess.PIPE)
-
- for line in proc.stdout:
- (key, _, value) = line.strip().partition("=")
- if key in ENV_KEYS:
- os.environ[key] = value
-
- proc.communicate()
-
-def java(classname, args, classpath, jvm_opts_list, logdir=None):
- java_home = os.environ.get("JAVA_HOME", None)
- if java_home:
- prg = os.path.join(java_home, "bin", "java")
- else:
- prg = which("java")
-
- commandline = [prg]
- commandline.extend(jvm_opts_list)
- commandline.append("-classpath")
- commandline.append(classpath)
- commandline.append(classname)
- commandline.extend(args)
- return runProcess(commandline, logdir)
-
-def jar(path):
- java_home = os.environ.get("JAVA_HOME", None)
- if java_home:
- prg = os.path.join(java_home, "bin", "jar")
- else:
- prg = which("jar")
-
- commandline = [prg]
- commandline.append("-xf")
- commandline.append(path)
- process = runProcess(commandline)
- process.wait()
-
-def is_exe(fpath):
- return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
-def which(program):
-
- fpath, fname = os.path.split(program)
- if fpath:
- if is_exe(program):
- return program
- else:
- for path in os.environ["PATH"].split(os.pathsep):
- path = path.strip('"')
- exe_file = os.path.join(path, program)
- if is_exe(exe_file):
- return exe_file
-
- return None
-
-def runProcess(commandline, logdir=None):
- """
- Run a process
- :param commandline: command line
- :return:the return code
- """
- global finished
- debug ("Executing : %s" % commandline)
- timestr = time.strftime("metadata.%Y%m%d-%H%M%S")
- stdoutFile = None
- stderrFile = None
- if logdir:
- stdoutFile = open(os.path.join(logdir, timestr + ".out"), "w")
- stderrFile = open(os.path.join(logdir,timestr + ".err"), "w")
- return subprocess.Popen(commandline, stdout=stdoutFile, stderr=stderrFile)
-
-def print_output(name, src, toStdErr):
- """
- Relay the output stream to stdout line by line
- :param name:
- :param src: source stream
- :param toStdErr: flag set if stderr is to be the dest
- :return:
- """
-
- global needPassword
- debug ("starting printer for %s" % name )
- line = ""
- while not finished:
- (line, done) = read(src, line)
- if done:
- out(toStdErr, line + "\n")
- flush(toStdErr)
- if line.find("Enter password for") >= 0:
- needPassword = True
- line = ""
- out(toStdErr, line)
- # closedown: read remainder of stream
- c = src.read(1)
- while c!="" :
- c = c.decode('utf-8')
- out(toStdErr, c)
- if c == "\n":
- flush(toStdErr)
- c = src.read(1)
- flush(toStdErr)
- src.close()
-
-def read_input(name, exe):
- """
- Read input from stdin and send to process
- :param name:
- :param process: process to send input to
- :return:
- """
- global needPassword
- debug ("starting reader for %s" % name )
- while not finished:
- if needPassword:
- needPassword = False
- if sys.stdin.isatty():
- cred = getpass.getpass()
- else:
- cred = sys.stdin.readline().rstrip()
- exe.stdin.write(cred + "\n")
-
-def debug(text):
- if DEBUG: print '[DEBUG] ' + text
-
-
-def error(text):
- print '[ERROR] ' + text
- sys.stdout.flush()
-
-def info(text):
- print text
- sys.stdout.flush()
-
-
-def out(toStdErr, text) :
- """
- Write to one of the system output channels.
- This action does not add newlines. If you want that: write them yourself
- :param toStdErr: flag set if stderr is to be the dest
- :param text: text to write.
- :return:
- """
- if toStdErr:
- sys.stderr.write(text)
- else:
- sys.stdout.write(text)
-
-def flush(toStdErr) :
- """
- Flush the output stream
- :param toStdErr: flag set if stderr is to be the dest
- :return:
- """
- if toStdErr:
- sys.stderr.flush()
- else:
- sys.stdout.flush()
-
-def read(pipe, line):
- """
- read a char, append to the listing if there is a char that is not \n
- :param pipe: pipe to read from
- :param line: line being built up
- :return: (the potentially updated line, flag indicating newline reached)
- """
-
- c = pipe.read(1)
- if c != "":
- o = c.decode('utf-8')
- if o != '\n':
- line += o
- return line, False
- else:
- return line, True
- else:
- return line, False
-
-def writePid(metadata_pid_file, process):
- f = open(metadata_pid_file, 'w')
- f.write(str(process.pid))
- f.close()
-
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/metadata_start.py
----------------------------------------------------------------------
diff --git a/src/bin/metadata_start.py b/src/bin/metadata_start.py
deleted file mode 100755
index a111bed..0000000
--- a/src/bin/metadata_start.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-import sys
-import traceback
-
-import metadata_config as mc
-
-METADATA_LOG_OPTS="-Dmetadata.log.dir=%s -Dmetadata.log.file=application.log"
-METADATA_COMMAND_OPTS="-Dmetadata.home=%s"
-METADATA_CONFIG_OPTS="-Dmetadata.conf=%s"
-DEFAULT_JVM_OPTS="-Xmx1024m"
-
-def main():
-
- metadata_home = mc.metadataDir()
- confdir = mc.dirMustExist(mc.confDir(metadata_home))
- mc.executeEnvSh(confdir)
- logdir = mc.dirMustExist(mc.logDir(metadata_home))
-
- #create sys property for conf dirs
- jvm_opts_list = (METADATA_LOG_OPTS % logdir).split()
-
- cmd_opts = (METADATA_COMMAND_OPTS % metadata_home)
- jvm_opts_list.extend(cmd_opts.split())
-
- config_opts = (METADATA_CONFIG_OPTS % confdir)
- jvm_opts_list.extend(config_opts.split())
-
- default_jvm_opts = DEFAULT_JVM_OPTS
- metadata_jvm_opts = os.environ.get(mc.METADATA_OPTS, default_jvm_opts)
- jvm_opts_list.extend(metadata_jvm_opts.split())
-
- #expand web app dir
- web_app_dir = mc.webAppDir(metadata_home)
- mc.expandWebApp(metadata_home)
-
- p = os.pathsep
- metadata_classpath = confdir + p \
- + os.path.join(web_app_dir, "metadata", "WEB-INF", "classes" ) + p \
- + os.path.join(web_app_dir, "metadata", "WEB-INF", "lib", "*" ) + p \
- + os.path.join(metadata_home, "libext", "*")
-
- metadata_pid_file = mc.pidFile(metadata_home)
-
- if os.path.isfile(metadata_pid_file):
- print "%s already exists, exiting" % metadata_pid_file
- sys.exit()
-
- args = ["-app", os.path.join(web_app_dir, "metadata")]
- args.extend(sys.argv[1:])
-
- process = mc.java("org.apache.hadoop.metadata.Main", args, metadata_classpath, jvm_opts_list, logdir)
- mc.writePid(metadata_pid_file, process)
-
- print "Metadata Server started!!!\n"
-
-if __name__ == '__main__':
- try:
- returncode = main()
- except Exception as e:
- print "Exception: %s " % str(e)
- print traceback.format_exc()
- returncode = -1
-
- sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/src/bin/metadata_stop.py
----------------------------------------------------------------------
diff --git a/src/bin/metadata_stop.py b/src/bin/metadata_stop.py
deleted file mode 100755
index 192ebf2..0000000
--- a/src/bin/metadata_stop.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-from signal import SIGTERM
-import sys
-import traceback
-
-import metadata_config as mc
-
-def main():
-
- metadata_home = mc.metadataDir()
- confdir = mc.dirMustExist(mc.confDir(metadata_home))
- mc.executeEnvSh(confdir)
- piddir = mc.dirMustExist(mc.logDir(metadata_home))
-
- metadata_pid_file = mc.pidFile(metadata_home)
-
- try:
- pf = file(metadata_pid_file, 'r')
- pid = int(pf.read().strip())
- pf.close()
- except:
- pid = None
-
- if not pid:
- sys.stderr.write("No process ID file found. Server not running?\n")
- return
-
- os.kill(pid, SIGTERM)
-
- # assuming kill worked since process check on windows is more involved...
- if os.path.exists(metadata_pid_file):
- os.remove(metadata_pid_file)
-
-if __name__ == '__main__':
- try:
- returncode = main()
- except Exception as e:
- print "Exception: %s " % str(e)
- print traceback.format_exc()
- returncode = -1
-
- sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala b/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala
new file mode 100755
index 0000000..7bc52e3
--- /dev/null
+++ b/tools/src/main/scala/org/apache/atlas/tools/dsl/DynamicTypedStruct.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.dsl
+
+import org.apache.atlas.typesystem.ITypedStruct
+import org.apache.atlas.typesystem.types.{StructType, TypeSystem}
+
+import scala.language.dynamics
+
+class DynamicTypedStruct(val ts: ITypedStruct) extends Dynamic {
+ def selectDynamic(name: String) = ts.get(name)
+
+ def updateDynamic(name: String)(value: Any) {
+ var value1 = value
+ if (value != null && value.isInstanceOf[DynamicTypedStruct]) {
+ value1 = value.asInstanceOf[DynamicTypedStruct].ts
+ }
+ ts.set(name, value1)
+ }
+
+ def dataType = TypeSystem.getInstance().getDataType(classOf[StructType], ts.getTypeName)
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala b/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala
new file mode 100755
index 0000000..b7b7e0c
--- /dev/null
+++ b/tools/src/main/scala/org/apache/atlas/tools/dsl/package.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools
+
+import org.apache.atlas.repository.memory.MemRepository
+import org.apache.atlas.typesystem.json.{BigDecimalSerializer, BigIntegerSerializer, Serialization, TypedStructSerializer}
+import org.apache.atlas.typesystem.persistence.StructInstance
+import org.apache.atlas.typesystem.types._
+import org.apache.atlas.typesystem.{IStruct, ITypedStruct}
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.{write => swrite}
+
+import scala.collection.JavaConversions._
+import scala.language.implicitConversions
+
+package object dsl {
+
+ val defFormat = new DefaultFormats {
+ override protected def dateFormatter = TypeSystem.getInstance().getDateFormat;
+
+ override val typeHints = NoTypeHints
+ }
+
+ implicit val formats = defFormat + new TypedStructSerializer +
+ new BigDecimalSerializer + new BigIntegerSerializer
+ val BOOLEAN_TYPE = DataTypes.BOOLEAN_TYPE
+ val BYTE_TYPE = DataTypes.BYTE_TYPE
+ val SHORT_TYPE = DataTypes.SHORT_TYPE
+ val INT_TYPE = DataTypes.INT_TYPE
+ val LONG_TYPE = DataTypes.LONG_TYPE
+ val FLOAT_TYPE = DataTypes.FLOAT_TYPE
+ val DOUBLE_TYPE = DataTypes.DOUBLE_TYPE
+ val BIGINT_TYPE = DataTypes.BIGINTEGER_TYPE
+ val BIGDECIMAL_TYPE = DataTypes.BIGDECIMAL_TYPE
+ val DATE_TYPE = DataTypes.DATE_TYPE
+ val STRING_TYPE = DataTypes.STRING_TYPE
+ val ATTR_OPTIONAL = Multiplicity.OPTIONAL
+ val ATTR_REQUIRED = Multiplicity.REQUIRED
+
+ def repo = new MemRepository(ts)
+
+ def arrayType(dT: IDataType[_]) = ts.defineArrayType(dT)
+
+ def mapType(kT: IDataType[_], vT: IDataType[_]) = ts.defineMapType(kT, vT)
+
+ def attrDef(name: String, dT: IDataType[_],
+ m: Multiplicity = Multiplicity.OPTIONAL,
+ isComposite: Boolean = false,
+ reverseAttributeName: String = null) = {
+ require(name != null)
+ require(dT != null)
+ new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName)
+ }
+
+ def listTypes = (ts.getTypeNames -- ts.getCoreTypes).sorted.toList.mkString("[", ",", "]")
+
+ def ts = TypeSystem.getInstance
+
+ def defineStructType(name: String, attrDef: AttributeDefinition*) = {
+ require(name != null)
+ ts.defineStructType(name, false, attrDef: _*)
+ }
+
+ def createInstance(typeName: String, jsonStr: String)(implicit formats: Formats) = {
+ val j = parse(jsonStr)
+ assert(j.isInstanceOf[JObject])
+ var j1 = j.asInstanceOf[JObject]
+ j1 = JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(typeName)) :: j1.obj)
+ new DynamicTypedStruct(Extraction.extract[StructInstance](j1))
+ }
+
+ def createInstance(typeName: String) = {
+ new DynamicTypedStruct(
+ ts.getDataType(classOf[StructType], typeName).asInstanceOf[IConstructableType[IStruct, ITypedStruct]].createInstance())
+ }
+
+ implicit def dynTypedStructToTypedStruct(s: DynamicTypedStruct) = s.ts
+
+ implicit def dynTypedStructToJson(s: DynamicTypedStruct)(implicit formats: Formats) = {
+ Extraction.decompose(s.ts)(formats)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala
new file mode 100755
index 0000000..7b98d77
--- /dev/null
+++ b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/Main.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.simpleserver
+
+import akka.actor.{ActorSystem, Props}
+import akka.io.IO
+import com.typesafe.config.ConfigFactory
+import org.apache.atlas.repository.memory.MemRepository
+import org.apache.atlas.typesystem.types.TypeSystem
+import spray.can.Http
+
+/**
+ * A Simple Spray based server to test the TypeSystem and MemRepository.
+ *
+ * @example {{{
+ * -- Using the [[ https://github.com/jakubroztocil/httpie Httpie tool]]
+ *
+ * http GET localhost:9140/listTypeNames
+ * pbpaste | http PUT localhost:9140/defineTypes
+ * http GET localhost:9140/typeDetails typeNames:='["Department", "Person", "Manager"]'
+ *
+ * pbpaste | http PUT localhost:9140/createInstance
+ * pbpaste | http GET localhost:9140/getInstance
+ * }}}
+ *
+ * - On the Mac, pbpaste makes available what is copied to clipboard. Copy contents of resources/sampleTypes.json
+ * - for createInstance resources/sampleInstance.json is an example
+ * - for getInstance send an Id back, you can copy the output from createInstance.
+ *
+ */
+object Main extends App {
+ val config = ConfigFactory.load()
+ val host = config.getString("http.host")
+ val port = config.getInt("http.port")
+
+ implicit val system = ActorSystem("atlasservice")
+
+ val typSys = TypeSystem.getInstance()
+ val memRepo = new MemRepository(typSys)
+
+ val api = system.actorOf(Props(new RestInterface(typSys, memRepo)), "httpInterface")
+ IO(Http) ! Http.Bind(listener = api, interface = host, port = port)
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala
new file mode 100755
index 0000000..0429814
--- /dev/null
+++ b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/MetadataActor.scala
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.simpleserver
+
+import akka.actor._
+import akka.util.Timeout
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.repository.memory.MemRepository
+import org.apache.atlas.typesystem.json._
+import org.apache.atlas.typesystem.persistence.Id
+import org.apache.atlas.typesystem.types._
+import org.apache.atlas.typesystem.{ITypedReferenceableInstance, TypesDef}
+import org.json4s.{Formats, NoTypeHints}
+import spray.httpx.Json4sSupport
+
+import scala.concurrent.duration._
+
+
+class MetadataActor(val typeSystem: TypeSystem, val memRepository: MemRepository) extends Actor with ActorLogging {
+
+ import org.apache.atlas.tools.simpleserver.MetadataProtocol._
+
+ import scala.collection.JavaConversions._
+ import scala.language.postfixOps
+ implicit val timeout = Timeout(5 seconds)
+
+
+ def receive = {
+ case ListTypeNames() =>
+ sender ! TypeNames(typeSystem.getTypeNames.toList)
+
+ case GetTypeDetails(typeNames) =>
+ val typesDef = TypesSerialization.convertToTypesDef(typeSystem, (d: IDataType[_]) => typeNames.contains(d.getName))
+ sender ! TypeDetails(typesDef)
+
+ case DefineTypes(typesDef: TypesDef) =>
+ typesDef.enumTypes.foreach(typeSystem.defineEnumType(_))
+
+ typeSystem.defineTypes(ImmutableList.copyOf(typesDef.structTypes.toArray),
+ ImmutableList.copyOf(typesDef.traitTypes.toArray),
+ ImmutableList.copyOf(typesDef.classTypes.toArray))
+
+ var newTypes: List[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]] = Nil
+ typesDef.traitTypes.foreach { tDef =>
+ val nm = tDef.typeName
+ newTypes = newTypes :+
+ typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm)
+ }
+ typesDef.classTypes.foreach { tDef =>
+ val nm = tDef.typeName
+ newTypes = newTypes :+
+ typeSystem.getDataType(classOf[HierarchicalType[_ <: HierarchicalType[_ <: AnyRef, _], _]], nm)
+ }
+
+ memRepository.defineTypes(newTypes)
+
+ sender ! TypesCreated
+
+ case CreateInstance(i) =>
+ val r = memRepository.create(i)
+ sender ! InstanceCreated(r.getId)
+
+ case GetInstance(id) =>
+ val r = memRepository.get(id)
+ sender ! InstanceDetails(r)
+ }
+
+}
+
+object MetadataProtocol {
+
+ case class ListTypeNames()
+
+ case class TypeNames(typeNames: List[String])
+
+ case class GetTypeDetails(typeNames: List[String])
+
+ case class TypeDetails(types: TypesDef)
+
+ case class DefineTypes(types: TypesDef)
+
+ case class TypesCreated()
+
+ case class CreateInstance(i: ITypedReferenceableInstance)
+
+ case class InstanceCreated(id: Id)
+
+ case class GetInstance(id: Id)
+
+ case class InstanceDetails(i: ITypedReferenceableInstance)
+
+}
+
+
+trait Json4sProtocol extends Json4sSupport {
+ val typeSystem: TypeSystem
+ val memRepository: MemRepository
+
+ implicit def json4sFormats: Formats =
+ org.json4s.native.Serialization.formats(NoTypeHints) + new MultiplicitySerializer +
+ new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer +
+ new BigDecimalSerializer + new BigIntegerSerializer + new IdSerializer
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala
new file mode 100755
index 0000000..6a48165
--- /dev/null
+++ b/tools/src/main/scala/org/apache/atlas/tools/simpleserver/RestInterface.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.simpleserver
+
+import akka.actor._
+import akka.util.Timeout
+import org.apache.atlas.repository.memory.MemRepository
+import org.apache.atlas.typesystem.persistence.Id
+import org.apache.atlas.typesystem.types.TypeSystem
+import org.apache.atlas.typesystem.{ITypedReferenceableInstance, TypesDef}
+import spray.http.StatusCodes
+import spray.routing._
+
+import scala.concurrent.duration._
+
+class Responder(val typeSystem: TypeSystem, val memRepository : MemRepository,
+ requestContext:RequestContext, mdSvc:ActorRef) extends Actor with Json4sProtocol with ActorLogging {
+ import org.apache.atlas.tools.simpleserver.MetadataProtocol._
+
+ def receive = {
+
+ case typNames:TypeNames =>
+ requestContext.complete(StatusCodes.OK, typNames)
+ self ! PoisonPill
+
+ case tD:TypeDetails =>
+ requestContext.complete(StatusCodes.OK, tD)
+ self ! PoisonPill
+
+ case TypesCreated =>
+ requestContext.complete(StatusCodes.OK)
+ self ! PoisonPill
+
+ case InstanceCreated(id) =>
+ requestContext.complete(StatusCodes.OK, id)
+
+ case InstanceDetails(i) =>
+ requestContext.complete(StatusCodes.OK, i)
+ }
+}
+
+class RestInterface(val typeSystem: TypeSystem, val memRepository : MemRepository) extends HttpServiceActor
+with RestApi {
+ def receive = runRoute(routes)
+}
+
+
+trait RestApi extends HttpService with Json4sProtocol with ActorLogging { actor: Actor =>
+ import MetadataProtocol._
+
+ import scala.concurrent.ExecutionContext.Implicits.global
+ import scala.language.postfixOps
+
+ val typeSystem : TypeSystem
+ val memRepository : MemRepository
+
+ implicit val timeout = Timeout(10 seconds)
+
+ import akka.pattern.{ask, pipe}
+
+ val mdSvc = context.actorOf(Props(new MetadataActor(typeSystem, memRepository)))
+
+ def routes: Route =
+
+ path("listTypeNames") {
+ get { requestContext =>
+ val responder: ActorRef = createResponder(requestContext)
+
+ pipe(mdSvc.ask(ListTypeNames))
+
+ mdSvc.ask(ListTypeNames()).pipeTo(responder)
+ }
+ } ~
+ path("typeDetails") {
+ get {
+ entity(as[GetTypeDetails]) { typeDetails => requestContext =>
+ val responder = createResponder(requestContext)
+ mdSvc.ask(typeDetails).pipeTo(responder)
+ }
+ }
+ } ~
+ path("defineTypes") {
+ put {
+ entity(as[TypesDef]) { typesDef => requestContext =>
+ val responder = createResponder(requestContext)
+ mdSvc.ask(DefineTypes(typesDef)).pipeTo(responder)
+ }
+ }
+ } ~
+ path("createInstance") {
+ put {
+ entity(as[ITypedReferenceableInstance]) { i => requestContext =>
+ val responder = createResponder(requestContext)
+ mdSvc.ask(CreateInstance(i)).pipeTo(responder)
+ }
+ }
+ } ~
+ path("getInstance") {
+ get {
+ entity(as[Id]) { id => requestContext =>
+ val responder = createResponder(requestContext)
+ mdSvc.ask(GetInstance(id)).pipeTo(responder)
+ }
+ }
+ }
+
+ def createResponder(requestContext:RequestContext) = {
+ context.actorOf(Props(new Responder(typeSystem, memRepository, requestContext, mdSvc)))
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala b/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala
new file mode 100755
index 0000000..67d4b37
--- /dev/null
+++ b/tools/src/main/scala/org/apache/atlas/tools/thrift/ThriftParser.scala
@@ -0,0 +1,664 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.thrift
+
+import org.apache.atlas.MetadataException
+import org.apache.atlas.typesystem.types.DataTypes
+
+import scala.util.parsing.combinator.lexical.StdLexical
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.{ImplicitConversions, PackratParsers}
+import scala.util.parsing.input.CharArrayReader._
+
+object BASE_TYPES extends Enumeration {
+ val STRING = Value("string")
+ val BINARY = Value("binary")
+ val SLIST = Value("slist")
+ val BOOLEAN = Value("bool")
+ val BYTE = Value("byte")
+ val I16 = Value("i16")
+ val I32 = Value("i32")
+ val I64 = Value("i64")
+ val DOUBLE = Value("double")
+
+ @throws[MetadataException]
+ def toPrimitiveTypeName(t : BASE_TYPES.Value) : String = t match {
+ case STRING => DataTypes.STRING_TYPE.getName
+ case SLIST => DataTypes.STRING_TYPE.getName
+ case BOOLEAN => DataTypes.BOOLEAN_TYPE.getName
+ case BYTE => DataTypes.BYTE_TYPE.getName
+ case I16 => DataTypes.SHORT_TYPE.getName
+ case I32 => DataTypes.INT_TYPE.getName
+ case I64 => DataTypes.LONG_TYPE.getName
+ case DOUBLE => DataTypes.DOUBLE_TYPE.getName
+ case _ => throw new MetadataException(s"Thrift BaseType ($t) not supported")
+ }
+}
+
+object THRIFT_LANG extends Enumeration {
+ val CPP = Value("cpp")
+ val PHP = Value("php")
+ val PY = Value("py")
+ val PERL = Value("perl")
+ val RUBY = Value("ruby")
+ val SMLTK_CAT = Value("smalltalk.category")
+ val SMLTK_PRE = Value("smalltalk.prefix")
+ val JAVA = Value("java")
+ val COCOA = Value("cocoa")
+ val XSD = Value("xsd")
+ val CSHARP = Value("csharp")
+ val STAR = Value("*")
+ val OTHER = Value("")
+}
+
+case class TypeAnnotation(name : String, value : String)
+case class CPPType(name : String)
+sealed trait FunctionType
+case class VoidType() extends FunctionType
+sealed trait FieldType extends FunctionType
+case class IdentifierType(name : String) extends FieldType
+case class BaseType(typ : BASE_TYPES.Value, typAnnotations :Option[List[TypeAnnotation]]) extends FieldType
+sealed trait ContainerType extends FieldType {
+ def typAnnotations :Option[List[TypeAnnotation]]
+}
+case class MapType(keyType : FieldType, valueType : FieldType,
+ cppType : Option[CPPType],
+ typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType
+case class SetType(elemType : FieldType,
+ cppType : Option[CPPType],
+ typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType
+case class ListType(elemType : FieldType,
+ cppType : Option[CPPType],
+ typAnnotations :Option[List[TypeAnnotation]]) extends ContainerType
+
+sealed trait ConstValue
+case class IntConstant(value : Int) extends ConstValue
+case class DoubleConstant(value : Double) extends ConstValue
+case class StringConstant(value : String) extends ConstValue
+case class IdConstant(value : String) extends ConstValue
+case class ConstantList(value : List[ConstValue]) extends ConstValue
+case class ConstantValuePair(first : ConstValue, second : ConstValue)
+case class ConstantMap(value : List[ConstantValuePair]) extends ConstValue
+
+case class ConstDef(fieldType : FieldType, id : String, value : ConstValue)
+
+case class TypeDef(name : String, fieldType : FieldType,
+ typAnnotations :Option[List[TypeAnnotation]])
+case class EnumValueDef(value : String, id : Option[IntConstant], typAnnotations :Option[List[TypeAnnotation]])
+case class EnumDef(name : String, enumValues : List[EnumValueDef], typAnnotations :Option[List[TypeAnnotation]])
+
+case class SEnumDef(name : String, enumValues : List[String], typAnnotations :Option[List[TypeAnnotation]])
+
+case class FieldDef(id : Option[IntConstant], requiredNess : Boolean, fieldType : FieldType, name : String,
+ fieldValue : Option[ConstValue], xsdOptional : Boolean, xsdNillable : Boolean,
+ xsdAttributes: Option[XsdAttributes],
+ typAnnotations :Option[List[TypeAnnotation]])
+
+case class XsdAttributes(fields : List[FieldDef])
+
+case class StructDef(name : String, xsdAll : Boolean, fields : List[FieldDef],
+ typAnnotations :Option[List[TypeAnnotation]])
+
+case class UnionDef(val name : String, val xsdAll : Boolean,
+ val fields : List[FieldDef],
+ val typAnnotations :Option[List[TypeAnnotation]])
+
+case class ExceptionDef(val name : String,
+ val fields : List[FieldDef],
+ val typAnnotations :Option[List[TypeAnnotation]])
+
+case class FunctionDef(oneway : Boolean, returnType : FunctionType, name : String, parameters : List[FieldDef],
+ throwFields : Option[List[FieldDef]], typAnnotations :Option[List[TypeAnnotation]])
+
+case class ServiceDef(name : String, superName : Option[String], functions : List[FunctionDef],
+ typAnnotations :Option[List[TypeAnnotation]])
+
+case class IncludeDef(value : String)
+case class CppIncludeDef(val value : String)
+case class NamespaceDef(lang : THRIFT_LANG.Value, name : String, otherLang : Option[String] = None)
+
+case class ThriftDef(val includes : List[IncludeDef],
+ val cppIncludes : List[CppIncludeDef],
+ val namespaces : List[NamespaceDef],
+ val constants : List[ConstDef],
+ val typedefs : List[TypeDef],
+ val enums : List[EnumDef],
+ val senums : List[SEnumDef],
+ val structs : List[StructDef],
+ val unions : List[UnionDef],
+ val xceptions : List[ExceptionDef],
+ val services : List[ServiceDef]) {
+
+ def this() = this(List(), List(), List(), List(), List(), List(), List(),
+ List(), List(), List(), List())
+
+ def this(a : IncludeDef) = this(a :: Nil, List(), List(), List(), List(), List(), List(),
+ List(), List(), List(), List())
+ def this(a : CppIncludeDef) = this(List(), a :: Nil, List(), List(), List(), List(), List(), List(),
+ List(), List(), List())
+ def this(a : NamespaceDef) = this(List(), List(), a :: Nil, List(), List(), List(), List(), List(), List(),
+ List(), List())
+ def this(a : ConstDef) = this(List(), List(), List(), a :: Nil, List(), List(), List(), List(), List(), List(),
+ List())
+ def this(a : TypeDef) = this(List(), List(), List(), List(), a :: Nil, List(), List(), List(), List(), List(), List())
+ def this(a : EnumDef) = this(List(), List(), List(), List(), List(), a :: Nil, List(), List(),
+ List(), List(), List())
+ def this(a : SEnumDef) = this(List(), List(), List(), List(), List(), List(), a :: Nil, List(),
+ List(), List(), List())
+ def this(a : StructDef) = this(List(), List(), List(), List(), List(), List(), List(), a :: Nil,
+ List(), List(), List())
+ def this(a : UnionDef) = this(List(), List(), List(), List(), List(), List(), List(),
+ List(), a :: Nil, List(), List())
+ def this(a : ExceptionDef) = this(List(), List(), List(), List(), List(), List(), List(),
+ List(), List(), a :: Nil, List())
+ def this(a : ServiceDef) = this(List(), List(), List(), List(), List(), List(), List(),
+ List(), List(), List(), a :: Nil)
+
+
+ def plus(a : IncludeDef) = ThriftDef(includes.+:(a), cppIncludes, namespaces, constants, typedefs, enums, senums,
+ structs, unions, xceptions, services)
+ def plus(a : CppIncludeDef) = ThriftDef(includes, cppIncludes.+:(a), namespaces, constants, typedefs, enums, senums,
+ structs, unions, xceptions, services)
+ def plus(a : NamespaceDef) = ThriftDef(includes, cppIncludes, namespaces.+:(a), constants, typedefs, enums, senums,
+ structs, unions, xceptions, services)
+ def plus(a : ConstDef) = ThriftDef(includes, cppIncludes, namespaces, constants.+:(a), typedefs, enums, senums,
+ structs, unions, xceptions, services)
+ def plus(a : TypeDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs.+:(a), enums, senums,
+ structs, unions, xceptions, services)
+ def plus(a : EnumDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums.+:(a), senums,
+ structs, unions, xceptions, services)
+ def plus(a : SEnumDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums.+:(a),
+ structs, unions, xceptions, services)
+ def plus(a : StructDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
+ structs.+:(a), unions, xceptions, services)
+ def plus(a : UnionDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
+ structs, unions.+:(a), xceptions, services)
+ def plus(a : ExceptionDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
+ structs, unions, xceptions.+:(a), services)
+ def plus(a : ServiceDef) = ThriftDef(includes, cppIncludes, namespaces, constants, typedefs, enums, senums,
+ structs, unions, xceptions, services.+:(a))
+ def plus(a : ThriftDef) = ThriftDef(includes ::: a.includes,
+ cppIncludes ::: a.cppIncludes,
+ namespaces ::: a.namespaces,
+ constants ::: a.constants,
+ typedefs ::: a.typedefs,
+ enums ::: a.enums,
+ senums ::: a.senums,
+ structs ::: a.structs,
+ unions ::: a.unions,
+ xceptions ::: a.xceptions,
+ services ::: a.services)
+
+
+}
+
+trait ThriftKeywords {
+ this : StandardTokenParsers =>
+
+ import scala.language.implicitConversions
+
+ protected case class Keyword(str: String)
+
+ protected implicit def asParser(k: Keyword): Parser[String] = k.str
+
+ protected val LPAREN = Keyword("(")
+ protected val RPAREN = Keyword(")")
+ protected val EQ = Keyword("=")
+ protected val CPP_TYPE = Keyword("cpp_type")
+ protected val LIST = Keyword("list")
+ protected val LT = Keyword("<")
+ protected val GT = Keyword(">")
+ protected val SET = Keyword("set")
+ protected val MAP = Keyword("map")
+ protected val STRING = Keyword("string")
+ protected val BINARY = Keyword("binary")
+ protected val SLIST = Keyword("slist")
+ protected val BOOL = Keyword("bool")
+ protected val BYTE = Keyword("byte")
+ protected val I16 = Keyword("i16")
+ protected val I32 = Keyword("i32")
+ protected val I64 = Keyword("i64")
+ protected val DOUBLE = Keyword("double")
+ protected val VOID = Keyword("void")
+ protected val REQUIRED = Keyword("required")
+ protected val OPTIONAL = Keyword("optional")
+ protected val COLON = Keyword(":")
+ protected val THROWS = Keyword("throws")
+ protected val ONEWAY = Keyword("oneway")
+ protected val EXTENDS = Keyword("extends")
+ protected val SERVICE = Keyword("service")
+ protected val EXCEPTION = Keyword("exception")
+ protected val LBRACKET = Keyword("{")
+ protected val RBRACKET = Keyword("}")
+ protected val XSD_ATTRS = Keyword("xsd_attributes")
+ protected val XSD_NILBLE = Keyword("xsd_nillable")
+ protected val XSD_OPT = Keyword("xsd_optional")
+ protected val XSD_ALL = Keyword("xsd_all")
+ protected val UNION = Keyword("union")
+ protected val LSQBRACKET = Keyword("[")
+ protected val RSQBRACKET = Keyword("]")
+ protected val CONST = Keyword("const")
+ protected val STRUCT = Keyword("struct")
+ protected val SENUM = Keyword("senum")
+ protected val ENUM = Keyword("enum")
+ protected val COMMA = Keyword(",")
+ protected val SEMICOLON = Keyword(";")
+ protected val TYPEDEF = Keyword("typedef")
+ protected val INCLUDE = Keyword("include")
+ protected val CPP_INCL = Keyword("cpp_include")
+ protected val NAMESPACE = Keyword("namespace")
+ protected val STAR = Keyword("*")
+ protected val CPP_NS = Keyword("cpp_namespace")
+ protected val PHP_NS = Keyword("php_namespace")
+ protected val PY_NS = Keyword("py_module")
+ protected val PERL_NS = Keyword("perl_package")
+ protected val RUBY_NS = Keyword("ruby_namespace")
+ protected val SMLTK_CAT = Keyword("smalltalk_category")
+ protected val SMLTK_PRE = Keyword("smalltalk_prefix")
+ protected val JAVA_NS = Keyword("java_package")
+ protected val COCOA_NS = Keyword("cocoa_package")
+ protected val XSD_NS = Keyword("xsd_namespace")
+ protected val CSHARP_NS = Keyword("csharp_namespace")
+
+ def isRequired(r : Option[String]) = r match {
+ case Some(REQUIRED) => true
+ case _ => false
+ }
+
+ def isXsdOptional(r : Option[String]) = r match {
+ case Some(XSD_OPT) => true
+ case _ => false
+ }
+
+ def isXsdNillable(r : Option[String]) = r match {
+ case Some(XSD_NILBLE) => true
+ case _ => false
+ }
+
+ def isXsdAll(r : Option[String]) = r match {
+ case Some(XSD_ALL) => true
+ case _ => false
+ }
+
+ def isOneWay(r : Option[String]) = r match {
+ case Some(ONEWAY) => true
+ case _ => false
+ }
+
+}
+
+trait ThriftTypeRules extends ThriftKeywords {
+ this : StandardTokenParsers =>
+
+ def containterType : Parser[ContainerType] = mapType | setType | listType
+
+ def setType = SET ~ cppType.? ~ LT ~ fieldType ~ GT ~ typeAnnotations.? ^^ {
+ case s ~ ct ~ lt ~ t ~ gt ~ tA => SetType(t, ct, tA)
+ }
+
+ def listType = LIST ~ LT ~ fieldType ~ GT ~ cppType.? ~ typeAnnotations.? ^^ {
+ case l ~ lt ~ t ~ gt ~ ct ~ tA => ListType(t, ct, tA)
+ }
+
+ def mapType = MAP ~ cppType.? ~ LT ~ fieldType ~ COMMA ~ fieldType ~ GT ~ typeAnnotations.? ^^ {
+ case s ~ ct ~ lt ~ kt ~ c ~ vt ~ gt ~ tA => MapType(kt, vt, ct, tA)
+ }
+
+ def cppType : Parser[CPPType] = CPP_TYPE ~ stringLit ^^ { case c ~ s => CPPType(s)}
+
+ def fieldType: Parser[FieldType] = ident ^^ {case i => IdentifierType(i)} |
+ baseType |
+ containterType
+
+ def baseType : Parser[BaseType] = simpleBaseType ~ typeAnnotations.? ^^ { case s ~ t => BaseType(s, t)}
+
+ def simpleBaseType : Parser[BASE_TYPES.Value] = STRING ^^^ BASE_TYPES.STRING |
+ BINARY ^^^ BASE_TYPES.BINARY |
+ SLIST ^^^ BASE_TYPES.SLIST |
+ BOOL ^^^ BASE_TYPES.BOOLEAN |
+ BYTE ^^^ BASE_TYPES.BYTE |
+ I16 ^^^ BASE_TYPES.I16 |
+ I32 ^^^ BASE_TYPES.I32 |
+ I64 ^^^ BASE_TYPES.I64 |
+ DOUBLE ^^^ BASE_TYPES.DOUBLE
+
+ def typeAnnotations : Parser[List[TypeAnnotation]] =
+ LPAREN ~ typeAnnotation.* ~ RPAREN ^^ { case l ~ t ~ r => t.toList}
+
+ def typeAnnotation : Parser[TypeAnnotation] =
+ (ident ~ EQ ~ stringLit ~ commaOrSemicolon.?) ^^ { case i ~ e ~ s ~ c => TypeAnnotation(i,s)}
+
+ def commaOrSemicolon : Parser[String] = COMMA | SEMICOLON
+
+}
+
+/**
+ * @todo extract Constant Rules into this Trait. This requires moving `hexConstant` here. But how to specify
+ * type of `HexConstant`, it is a Path dependent Type tied to lexical member of ThriftParser.
+ */
+trait ThriftConstantRules extends ThriftKeywords {
+ this: StandardTokenParsers =>
+
+ // def parseDouble(s: String) = try { Some(s.toDouble) } catch { case _ : Throwable => None }
+ //
+ // def constValue : Parser[ConstValue] = numericLit ^^ {
+ // case n => parseDouble(n) match {
+ // case Some(d) => DoubleConstant(d)
+ // case _ => IntConstant(n.toInt)
+ // }
+ // } |
+ // hexConstant ^^ { case h => IntConstant(Integer.parseInt(h, 16))} |
+ // stringLit ^^ { case s => StringConstant(s)} |
+ // ident ^^ { case i => IdConstant(i)} |
+ // constList |
+ // constMap
+ //
+ // def constValuePair = constValue ~ COLON ~ constValue ~ commaOrSemicolon.? ^^ {
+ // case k ~ c ~ v ~ cs => ConstantValuePair(k,v)
+ // }
+ //
+ // def constList = LSQBRACKET ~ (constValue <~ commaOrSemicolon).* ~ RSQBRACKET ^^ {
+ // case l ~ vs ~ r => ConstantList(vs)
+ // }
+ //
+ // def constMap = LBRACKET ~ constValuePair.* ~ RBRACKET ^^ {
+ // case l ~ ps ~ r => ConstantMap(ps)
+ // }
+}
+
+/**
+ * A Parser for Thrift definition scripts.
+ * Based on [[https://github.com/twitter/commons/blob/master/src/antlr/twitter/thrift/descriptors/AntlrThrift.g]].
+ * Definition is parsed into a [[org.apache.atlas.tools.thrift.ThriftDef ThriftDef]] structure.
+ *
+ * @example {{{
+ * var p = new ThriftParser
+ * var td : Option[ThriftDef] = p("""include "share/fb303/if/fb303.thrift"
+ * namespace java org.apache.hadoop.hive.metastore.api
+ * namespace php metastore
+ * namespace cpp Apache.Hadoop.Hive
+ * \""")
+ * }}}
+ *
+ * @todo doesn't traverse includes directives. Includes are parsed into
+ * [[org.apache.atlas.tools.thrift.IncludeDef IncludeDef]] structures
+ * but are not traversed.
+ * @todo mixing in [[scala.util.parsing.combinator.PackratParsers PackratParsers]] is a placeholder. Need to
+ * change specific grammar rules to `lazy val` and `Parser[Elem]` to `PackratParser[Elem]`. Will do based on
+ * performance analysis.
+ * @todo Error reporting
+ */
+class ThriftParser extends StandardTokenParsers with ThriftKeywords with ThriftTypeRules with PackratParsers {
+
+ import scala.language.higherKinds
+
+ private val reservedWordsDelims : Seq[String] =
+ this
+ .getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ private val (thriftreservedWords : Seq[String], thriftdelims : Seq[String]) =
+ reservedWordsDelims.partition(s => s.charAt(0).isLetter)
+
+ override val lexical = new ThriftLexer(thriftreservedWords, thriftdelims)
+
+ import lexical.HexConstant
+ /** A parser which matches a hex constant */
+ def hexConstant: Parser[String] =
+ elem("string literal", _.isInstanceOf[HexConstant]) ^^ (_.chars)
+
+ def apply(input: String): Option[ThriftDef] = {
+ phrase(program)(new lexical.Scanner(input)) match {
+ case Success(r, x) => Some(r)
+ case Failure(m, x) => {
+ None
+ }
+ case Error(m, x) => {
+ None
+ }
+ }
+ }
+
+ def program = headers ~ definitions ^^ { case h ~ d => h plus d}
+
+ def headers = header.* ^^ { case l => l.foldRight(new ThriftDef)((a,t) => t plus a)}
+
+ def header = INCLUDE ~> stringLit ^^ { case s => new ThriftDef(IncludeDef(s))} |
+ CPP_INCL ~> stringLit ^^ { case s => new ThriftDef(CppIncludeDef(s))} |
+ NAMESPACE ~ ident ~ ident ^^ { case ns ~ t ~ n => new ThriftDef(NamespaceDef(THRIFT_LANG.OTHER, n, Some(t)))} |
+ NAMESPACE ~ STAR ~ ident ^^ { case ns ~ s ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.STAR, i))} |
+ CPP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.CPP, i))} |
+ PHP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PHP, i))} |
+ PY_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PY, i))} |
+ PERL_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.PERL, i))} |
+ RUBY_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.RUBY, i))} |
+ SMLTK_CAT ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.SMLTK_CAT, i))} |
+ SMLTK_PRE ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.SMLTK_PRE, i))} |
+ JAVA_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.JAVA, i))} |
+ COCOA_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.COCOA, i))} |
+ XSD_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.XSD, i))} |
+ CSHARP_NS ~ ident ^^ { case ns ~ i => new ThriftDef(NamespaceDef(THRIFT_LANG.CSHARP, i))}
+
+ def definitions : Parser[ThriftDef] = definition.* ^^ {
+ case l => l.foldRight(new ThriftDef)((a,t) => t plus a)
+ }
+
+ def definition : Parser[ThriftDef] = const ^^ { case c => new ThriftDef(c)} |
+ typeDefinition |
+ service ^^ { case s => new ThriftDef(s)}
+
+
+ def typeDefinition : Parser[ThriftDef] = (typedef ^^ {case t => new ThriftDef(t)} |
+ enum ^^ {case e => new ThriftDef(e)} |
+ senum ^^ {case e => new ThriftDef(e)} |
+ struct ^^ {case e => new ThriftDef(e)} |
+ union ^^ {case e => new ThriftDef(e)} |
+ xception ^^ {case e => new ThriftDef(e)}
+ )
+
+ def typedef : Parser[TypeDef] = TYPEDEF ~ fieldType ~ ident ~ typeAnnotations.? ^^ {
+ case t ~ f ~ i ~ tA => TypeDef(i, f, tA)
+ }
+
+ def enum : Parser[EnumDef] = ENUM ~ ident ~ LBRACKET ~ enumDef.* ~ RBRACKET ~ typeAnnotations.? ^^ {
+ case e ~ i ~ l ~ ed ~ r ~ t => EnumDef(i, ed.toList, t)
+ }
+
+ def enumDef : Parser[EnumValueDef] = ident ~ EQ ~ numericLit ~ typeAnnotations.? ~ commaOrSemicolon.? ^^ {
+ case i ~ e ~ n ~ t ~ c => EnumValueDef(i, Some(IntConstant(n.toInt)), t)
+ }
+
+ def senum : Parser[SEnumDef] = SENUM ~ ident ~ LBRACKET ~ senumDef.* ~ RBRACKET ~ typeAnnotations.? ^^ {
+ case se ~ i ~ l ~ sed ~ r ~ t => SEnumDef(i, sed.toList, t)
+ }
+
+ def senumDef : Parser[String] = stringLit <~ commaOrSemicolon.?
+
+ def service : Parser[ServiceDef] = SERVICE ~ ident ~ extnds.? ~ LBRACKET ~ function.* ~
+ RBRACKET ~ typeAnnotations.? ^^ {
+ case s ~ i ~ e ~ lb ~ fs ~ rb ~ tA => ServiceDef(i, e, fs, tA)
+ }
+
+ def extnds : Parser[String] = EXTENDS ~> ident
+
+ def function : Parser[FunctionDef] = ONEWAY.? ~ functionType ~ ident ~ LPAREN ~ field.* ~ RPAREN ~ throwz.? ~
+ typeAnnotations.? ~ commaOrSemicolon.? ^^ {
+ case o ~ fT ~ i ~ lp ~ fs ~ rp ~ th ~ tA ~ cS => FunctionDef(isOneWay(o), fT, i, fs, th, tA)
+ }
+
+ def throwz : Parser[List[FieldDef]] = THROWS ~ LPAREN ~ field.* ~ RPAREN ^^ {
+ case t ~ l ~ fs ~ r => fs.toList
+ }
+
+ def functionType : Parser[FunctionType] = VOID ^^^ VoidType() | fieldType
+
+ def xception : Parser[ExceptionDef] = EXCEPTION ~ ident ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ {
+ case s ~ i ~ lb ~ fs ~ rb ~ tA => ExceptionDef(i, fs.toList, tA)
+ }
+
+ def union : Parser[UnionDef] = UNION ~ ident ~ XSD_ALL.? ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ {
+ case s ~ i ~ xA ~ lb ~ fs ~ rb ~ tA => UnionDef(i, isXsdAll(xA), fs.toList, tA)
+ }
+
+ def struct : Parser[StructDef] = STRUCT ~ ident ~ XSD_ALL.? ~ LBRACKET ~ field.* ~ RBRACKET ~ typeAnnotations.? ^^ {
+ case s ~ i ~ xA ~ lb ~ fs ~ rb ~ tA => StructDef(i, isXsdAll(xA), fs.toList, tA)
+ }
+
+ def field : Parser[FieldDef] = fieldIdentifier.? ~ fieldRequiredness.? ~ fieldType ~ ident ~ fieldValue.? ~
+ XSD_OPT.? ~ XSD_NILBLE.? ~ xsdAttributes.? ~ typeAnnotations.? ~ commaOrSemicolon.? ^^ {
+ case fi ~ fr ~ ft ~id ~ fv ~ xo ~ xn ~ xa ~ tA ~ cS => FieldDef(
+ fi,
+ isRequired(fr),
+ ft,
+ id,
+ fv,
+ isXsdOptional(xo),
+ isXsdNillable(xn),
+ xa,
+ tA
+ )
+ }
+
+ def xsdAttributes : Parser[XsdAttributes] = XSD_ATTRS ~ LBRACKET ~ field.* ~ RBRACKET ^^ {
+ case x ~ l ~ f ~ r => XsdAttributes(f)
+ }
+
+ def fieldValue = EQ ~> constValue
+
+ def fieldRequiredness : Parser[String] = REQUIRED | OPTIONAL
+
+ def fieldIdentifier : Parser[IntConstant] = numericLit <~ COLON ^^ {
+ case n => IntConstant(n.toInt)
+ }
+
+ def const : Parser[ConstDef] = CONST ~ fieldType ~ ident ~ EQ ~ constValue ~ commaOrSemicolon.? ^^ {
+ case c ~ fT ~ i ~ e ~ cV ~ cS => ConstDef(fT, i, cV)
+ }
+
+ def parseDouble(s: String) = try { Some(s.toDouble) } catch { case _ : Throwable => None }
+
+ def constValue : Parser[ConstValue] = numericLit ^^ {
+ case n => parseDouble(n) match {
+ case Some(d) => DoubleConstant(d)
+ case _ => IntConstant(n.toInt)
+ }
+ } |
+ hexConstant ^^ { case h => IntConstant(Integer.parseInt(h, 16))} |
+ stringLit ^^ { case s => StringConstant(s)} |
+ ident ^^ { case i => IdConstant(i)} |
+ constList |
+ constMap
+
+ def constValuePair = constValue ~ COLON ~ constValue ~ commaOrSemicolon.? ^^ {
+ case k ~ c ~ v ~ cs => ConstantValuePair(k,v)
+ }
+
+ def constList = LSQBRACKET ~ (constValue <~ commaOrSemicolon).* ~ RSQBRACKET ^^ {
+ case l ~ vs ~ r => ConstantList(vs)
+ }
+
+ def constMap = LBRACKET ~ constValuePair.* ~ RBRACKET ^^ {
+ case l ~ ps ~ r => ConstantMap(ps)
+ }
+}
+
+class ThriftLexer(val keywords: Seq[String], val delims : Seq[String]) extends StdLexical with ImplicitConversions {
+
+ case class HexConstant(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ case class StIdentifier(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ reserved ++= keywords
+
+ delimiters ++= delims
+
+ override lazy val token: Parser[Token] =
+ ( intConstant ^^ NumericLit
+ | hexConstant ^^ HexConstant
+ | dubConstant ^^ NumericLit
+ | identifier ^^ processIdent
+ | st_identifier ^^ StIdentifier
+ | string ^^ StringLit
+ | EofCh ^^^ EOF
+ | '\'' ~> failure("unclosed string literal")
+ | '"' ~> failure("unclosed string literal")
+ | delim
+ | failure("illegal character")
+ )
+
+ override def identChar = letter | elem('_')
+
+ def identifier = identChar ~ (identChar | digit | '.' ).* ^^
+ { case first ~ rest => (first :: rest).mkString }
+
+ def st_identChar = letter | elem('-')
+ def st_identifier = st_identChar ~ (st_identChar | digit | '.' | '_').* ^^
+ { case first ~ rest => (first :: rest).mkString("")}
+
+ override def whitespace: Parser[Any] =
+ ( whitespaceChar
+ | '/' ~ '*' ~ comment
+ | '/' ~ '/' ~ chrExcept(EofCh, '\n').*
+ | '#' ~ chrExcept(EofCh, '\n').*
+ | '/' ~ '*' ~ failure("unclosed comment")
+ ).*
+
+ protected override def comment: Parser[Any] = (
+ commentChar.* ~ '*' ~ '/'
+ )
+
+ protected def commentChar = chrExcept(EofCh, '*') | '*' ~ not('/')
+
+ def string = '\"' ~> chrExcept('\"', '\n', EofCh).* <~ '\"' ^^ { _ mkString "" } |
+ '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^ { _ mkString "" }
+
+ def zero: Parser[String] = '0' ^^^ "0"
+ def nonzero = elem("nonzero digit", d => d.isDigit && d != '0')
+ def sign = elem("sign character", d => d == '-' || d == '+')
+ def exponent = elem("exponent character", d => d == 'e' || d == 'E')
+
+
+ def intConstant = opt(sign) ~> zero | intList
+ def intList = opt(sign) ~ nonzero ~ rep(digit) ^^ {case s ~ x ~ y => (optString("", s) :: x :: y) mkString ""}
+ def fracPart = '.' ~> rep(digit) ^^ { "." + _ mkString "" }
+ def expPart = exponent ~ opt(sign) ~ rep1(digit) ^^ { case e ~ s ~ d =>
+ e.toString + optString("", s) + d.mkString("")
+ }
+
+ def dubConstant = opt(sign) ~ digit.* ~ fracPart ~ opt(expPart) ^^ { case s ~ i ~ f ~ e =>
+ optString("", s) + i + f + optString("", e)
+ }
+
+ val hexDigits = Set[Char]() ++ "0123456789abcdefABCDEF".toArray
+ def hexDigit = elem("hex digit", hexDigits.contains(_))
+
+ def hexConstant = '0' ~> 'x' ~> hexDigit.+ ^^ {case h => h.mkString("")}
+
+
+ private def optString[A](pre: String, a: Option[A]) = a match {
+ case Some(x) => pre + x.toString
+ case None => ""
+ }
+}
[38/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
new file mode 100755
index 0000000..c4db02a
--- /dev/null
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -0,0 +1,379 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.hook;
+
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
+import org.apache.atlas.hive.model.HiveDataModelGenerator;
+import org.apache.atlas.hive.model.HiveDataTypes;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.util.Map;
+
+public class HiveHookIT {
+ public static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HiveHookIT.class);
+
+ private static final String DGI_URL = "http://localhost:21000/";
+ private static final String CLUSTER_NAME = "test";
+ public static final String DEFAULT_DB = "default";
+ private Driver driver;
+ private MetadataServiceClient dgiCLient;
+ private SessionState ss;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ //Set-up hive session
+ HiveConf conf = getHiveConf();
+ driver = new Driver(conf);
+ ss = new SessionState(conf, System.getProperty("user.name"));
+ ss = SessionState.start(ss);
+ SessionState.setCurrentSessionState(ss);
+
+ dgiCLient = new MetadataServiceClient(DGI_URL);
+ }
+
+ private HiveConf getHiveConf() {
+ HiveConf hiveConf = new HiveConf(this.getClass());
+ hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
+ hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore");
+ hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
+ hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
+ hiveConf.set("hive.hook.dgi.synchronous", "true");
+ hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME);
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODE, true); //to not use hdfs
+ hiveConf.setVar(HiveConf.ConfVars.HIVETESTMODEPREFIX, "");
+ hiveConf.set("fs.pfile.impl", "org.apache.hadoop.fs.ProxyLocalFileSystem");
+ return hiveConf;
+ }
+
+ private void runCommand(String cmd) throws Exception {
+ ss.setCommandType(null);
+ driver.run(cmd);
+ }
+
+ @Test
+ public void testCreateDatabase() throws Exception {
+ String dbName = "db" + random();
+ runCommand("create database " + dbName + " WITH DBPROPERTIES ('p1'='v1', 'p2'='v2')");
+ String dbId = assertDatabaseIsRegistered(dbName);
+ Referenceable definition = dgiCLient.getEntity(dbId);
+ Map params = (Map) definition.get("parameters");
+ Assert.assertNotNull(params);
+ Assert.assertEquals(params.size(), 2);
+ Assert.assertEquals(params.get("p1"), "v1");
+
+ //There should be just one entity per dbname
+ runCommand("drop database " + dbName);
+ runCommand("create database " + dbName);
+ assertDatabaseIsRegistered(dbName);
+ }
+
+ private String dbName() {
+ return "db" + random();
+ }
+
+ private String createDatabase() throws Exception {
+ String dbName = dbName();
+ runCommand("create database " + dbName);
+ return dbName;
+ }
+
+ private String tableName() {
+ return "table" + random();
+ }
+
+ private String createTable() throws Exception {
+ return createTable(true);
+ }
+
+ private String createTable(boolean partition) throws Exception {
+ String tableName = tableName();
+ runCommand("create table " + tableName + "(id int, name string) comment 'table comment' "
+ + (partition ? " partitioned by(dt string)" : ""));
+ return tableName;
+ }
+
+ @Test
+ public void testCreateTable() throws Exception {
+ String tableName = tableName();
+ String dbName = createDatabase();
+ String colName = "col" + random();
+ runCommand("create table " + dbName + "." + tableName + "(" + colName + " int, name string)");
+ assertTableIsRegistered(dbName, tableName);
+ //there is only one instance of column registered
+ assertColumnIsRegistered(colName);
+
+ tableName = createTable();
+ String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
+ Referenceable tableRef = dgiCLient.getEntity(tableId);
+ Assert.assertEquals(tableRef.get("tableType"), TableType.MANAGED_TABLE.name());
+ Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment");
+ String entityName = HiveMetaStoreBridge.getTableName(CLUSTER_NAME, DEFAULT_DB, tableName);
+ Assert.assertEquals(tableRef.get(HiveDataModelGenerator.NAME), entityName);
+
+ final Id sdId = (Id) tableRef.get("sd");
+ Referenceable sdRef = dgiCLient.getEntity(sdId.id);
+ Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS),false);
+
+ //Create table where database doesn't exist, will create database instance as well
+ assertDatabaseIsRegistered(DEFAULT_DB);
+ }
+
+ private String assertColumnIsRegistered(String colName) throws Exception {
+ LOG.debug("Searching for column {}", colName);
+ String query = String.format("%s where name = '%s'", HiveDataTypes.HIVE_COLUMN.getName(), colName.toLowerCase());
+ return assertEntityIsRegistered(query, true);
+
+ }
+
+ @Test
+ public void testCTAS() throws Exception {
+ String tableName = createTable();
+ String ctasTableName = "table" + random();
+ String query = "create table " + ctasTableName + " as select * from " + tableName;
+ runCommand(query);
+
+ assertTableIsRegistered(DEFAULT_DB, ctasTableName);
+ assertProcessIsRegistered(query);
+ }
+
+ @Test
+ public void testCreateView() throws Exception {
+ String tableName = createTable();
+ String viewName = tableName();
+ String query = "create view " + viewName + " as select * from " + tableName;
+ runCommand(query);
+
+ assertTableIsRegistered(DEFAULT_DB, viewName);
+ assertProcessIsRegistered(query);
+ }
+
+ @Test
+ public void testLoadData() throws Exception {
+ String tableName = createTable(false);
+
+ String loadFile = file("load");
+ String query = "load data local inpath 'file://" + loadFile + "' into table " + tableName;
+ runCommand(query);
+
+ assertProcessIsRegistered(query);
+ }
+
+ @Test
+ public void testInsert() throws Exception {
+ String tableName = createTable();
+ String insertTableName = createTable();
+ String query = "insert into " + insertTableName + " partition(dt = '2015-01-01') select id, name from "
+ + tableName + " where dt = '2015-01-01'";
+
+ runCommand(query);
+ assertProcessIsRegistered(query);
+ assertPartitionIsRegistered(DEFAULT_DB, insertTableName, "2015-01-01");
+ }
+
+ private String random() {
+ return RandomStringUtils.randomAlphanumeric(10);
+ }
+
+ private String file(String tag) throws Exception {
+ String filename = "./target/" + tag + "-data-" + random();
+ File file = new File(filename);
+ file.createNewFile();
+ return file.getAbsolutePath();
+ }
+
+ private String mkdir(String tag) throws Exception {
+ String filename = "./target/" + tag + "-data-" + random();
+ File file = new File(filename);
+ file.mkdirs();
+ return file.getAbsolutePath();
+ }
+
+ @Test
+ public void testExportImport() throws Exception {
+ String tableName = createTable(false);
+
+ String filename = "pfile://" + mkdir("export");
+ String query = "export table " + tableName + " to \"" + filename + "\"";
+ runCommand(query);
+ assertProcessIsRegistered(query);
+
+ tableName = createTable(false);
+
+ query = "import table " + tableName + " from '" + filename + "'";
+ runCommand(query);
+ assertProcessIsRegistered(query);
+ }
+
+ @Test
+ public void testSelect() throws Exception {
+ String tableName = createTable();
+ String query = "select * from " + tableName;
+ runCommand(query);
+ assertProcessIsRegistered(query);
+
+ //single entity per query
+ query = "SELECT * from " + tableName.toUpperCase();
+ runCommand(query);
+ assertProcessIsRegistered(query);
+ }
+
+ @Test
+ public void testAlterTable() throws Exception {
+ String tableName = createTable();
+ String newName = tableName();
+ String query = "alter table " + tableName + " rename to " + newName;
+ runCommand(query);
+
+ assertTableIsRegistered(DEFAULT_DB, newName);
+ assertTableIsNotRegistered(DEFAULT_DB, tableName);
+ }
+
+ @Test
+ public void testAlterView() throws Exception {
+ String tableName = createTable();
+ String viewName = tableName();
+ String newName = tableName();
+ String query = "create view " + viewName + " as select * from " + tableName;
+ runCommand(query);
+
+ query = "alter view " + viewName + " rename to " + newName;
+ runCommand(query);
+
+ assertTableIsRegistered(DEFAULT_DB, newName);
+ assertTableIsNotRegistered(DEFAULT_DB, viewName);
+ }
+
+ private void assertProcessIsRegistered(String queryStr) throws Exception {
+// String dslQuery = String.format("%s where queryText = \"%s\"", HiveDataTypes.HIVE_PROCESS.getName(),
+// normalize(queryStr));
+// assertEntityIsRegistered(dslQuery, true);
+ //todo replace with DSL
+ String typeName = HiveDataTypes.HIVE_PROCESS.getName();
+ String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.queryText', \"%s\").toList()",
+ typeName, typeName, normalize(queryStr));
+ JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
+ JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertEquals(results.length(), 1);
+ }
+
+ private String normalize(String str) {
+ if (StringUtils.isEmpty(str)) {
+ return null;
+ }
+ return StringEscapeUtils.escapeJava(str.toLowerCase());
+ }
+
+ private String assertTableIsRegistered(String dbName, String tableName) throws Exception {
+ return assertTableIsRegistered(dbName, tableName, true);
+ }
+
+ private String assertTableIsNotRegistered(String dbName, String tableName) throws Exception {
+ return assertTableIsRegistered(dbName, tableName, false);
+ }
+
+ private String assertTableIsRegistered(String dbName, String tableName, boolean registered) throws Exception {
+ LOG.debug("Searching for table {}.{}", dbName, tableName);
+ String query = String.format("%s as t where tableName = '%s', db where name = '%s' and clusterName = '%s'"
+ + " select t", HiveDataTypes.HIVE_TABLE.getName(), tableName.toLowerCase(), dbName.toLowerCase(),
+ CLUSTER_NAME);
+ return assertEntityIsRegistered(query, registered);
+ }
+
+ private String assertDatabaseIsRegistered(String dbName) throws Exception {
+ LOG.debug("Searching for database {}", dbName);
+ String query = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(),
+ dbName.toLowerCase(), CLUSTER_NAME);
+ return assertEntityIsRegistered(query, true);
+ }
+
+ private void assertPartitionIsRegistered(String dbName, String tableName, String value) throws Exception {
+ String typeName = HiveDataTypes.HIVE_PARTITION.getName();
+ String dbType = HiveDataTypes.HIVE_DB.getName();
+ String tableType = HiveDataTypes.HIVE_TABLE.getName();
+
+ LOG.debug("Searching for partition of {}.{} with values {}", dbName, tableName, value);
+ //todo replace with DSL
+ String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
+ + "out('__%s.table').has('%s.tableName', '%s').out('__%s.db').has('%s.name', '%s')"
+ + ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, value, typeName,
+ tableType, tableName.toLowerCase(), tableType, dbType, dbName.toLowerCase(), dbType, CLUSTER_NAME);
+ JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
+ JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertEquals(results.length(), 1);
+ }
+
+ private String assertEntityIsRegistered(String dslQuery, boolean registered) throws Exception{
+ JSONArray results = dgiCLient.searchByDSL(dslQuery);
+ if (registered) {
+ Assert.assertEquals(results.length(), 1);
+ JSONObject row = results.getJSONObject(0);
+ if (row.has("$id$")) {
+ return row.getJSONObject("$id$").getString("id");
+ } else {
+ return row.getJSONObject("_col_0").getString("id");
+ }
+ } else {
+ Assert.assertEquals(results.length(), 0);
+ return null;
+ }
+ }
+
+ @Test
+ public void testLineage() throws Exception {
+ String table1 = createTable(false);
+
+ String db2 = createDatabase();
+ String table2 = tableName();
+
+ String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
+ runCommand(query);
+ String table1Id = assertTableIsRegistered(DEFAULT_DB, table1);
+ String table2Id = assertTableIsRegistered(db2, table2);
+
+ String datasetName = HiveMetaStoreBridge.getTableName(CLUSTER_NAME, db2, table2);
+ JSONObject response = dgiCLient.getInputGraph(datasetName);
+ JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
+ Assert.assertTrue(vertices.has(table1Id));
+ Assert.assertTrue(vertices.has(table2Id));
+
+ datasetName = HiveMetaStoreBridge.getTableName(CLUSTER_NAME, DEFAULT_DB, table1);
+ response = dgiCLient.getOutputGraph(datasetName);
+ vertices = response.getJSONObject("values").getJSONObject("vertices");
+ Assert.assertTrue(vertices.has(table1Id));
+ Assert.assertTrue(vertices.has(table2Id));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
new file mode 100755
index 0000000..6caa4fc
--- /dev/null
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.hook;
+
+import org.apache.atlas.PropertiesUtil;
+import org.apache.atlas.security.SecurityProperties;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.net.URL;
+import java.nio.file.Files;
+
+import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
+
+/**
+ * Perform all the necessary setup steps for client and server comm over SSL/Kerberos, but then don't estalish a
+ * kerberos user for the invocation. Need a separate use case since the Jersey layer cached the URL connection handler,
+ * which indirectly caches the kerberos delegation token.
+ */
+public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
+
+ private Driver driver;
+ private SessionState ss;
+ private TestSecureEmbeddedServer secureEmbeddedServer;
+ private String originalConf;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ //Set-up hive session
+ HiveConf conf = getHiveConf();
+ driver = new Driver(conf);
+ ss = new SessionState(conf, System.getProperty("user.name"));
+ ss = SessionState.start(ss);
+ SessionState.setCurrentSessionState(ss);
+
+ jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ String persistDir = null;
+ URL resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/");
+ if (resource != null) {
+ persistDir = resource.toURI().getPath();
+ }
+ // delete prior ssl-client.xml file
+ resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
+ if (resource != null) {
+ File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
+ if (sslClientFile != null && sslClientFile.exists()) {
+ sslClientFile.delete();
+ }
+ }
+ setupKDCAndPrincipals();
+ setupCredentials();
+
+ // client will actually only leverage subset of these properties
+ final PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.setProperty(TLS_ENABLED, true);
+ configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
+ configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
+ configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+ configuration.setProperty("atlas.http.authentication.type", "kerberos");
+ configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
+
+ configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
+
+ String confLocation = System.getProperty("atlas.conf");
+ URL url;
+ if (confLocation == null) {
+ url = PropertiesUtil.class.getResource("/application.properties");
+ } else {
+ url = new File(confLocation, "application.properties").toURI().toURL();
+ }
+ configuration.load(url);
+ configuration.setProperty(TLS_ENABLED, true);
+ configuration.setProperty("atlas.http.authentication.enabled", "true");
+ configuration.setProperty("atlas.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
+ configuration.setProperty("atlas.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
+ configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
+ "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
+
+ configuration.save(new FileWriter(persistDir + File.separator + "application.properties"));
+
+ secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
+ @Override
+ public PropertiesConfiguration getConfiguration() {
+ return configuration;
+ }
+ };
+ WebAppContext webapp = new WebAppContext();
+ webapp.setContextPath("/");
+ webapp.setWar(System.getProperty("user.dir") + getWarPath());
+ secureEmbeddedServer.getServer().setHandler(webapp);
+
+ // save original setting
+ originalConf = System.getProperty("atlas.conf");
+ System.setProperty("atlas.conf", persistDir);
+ secureEmbeddedServer.getServer().start();
+
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ if (secureEmbeddedServer != null) {
+ secureEmbeddedServer.getServer().stop();
+ }
+
+ if (kdc != null) {
+ kdc.stop();
+ }
+
+ if (originalConf != null) {
+ System.setProperty("atlas.conf", originalConf);
+ }
+ }
+
+ private void runCommand(final String cmd) throws Exception {
+ ss.setCommandType(null);
+ driver.run(cmd);
+ Assert.assertNotNull(driver.getErrorMsg());
+ Assert.assertTrue(driver.getErrorMsg().contains("Mechanism level: Failed to find any Kerberos tgt"));
+ }
+
+ @Test
+ public void testUnsecuredCreateDatabase() throws Exception {
+ String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
+ runCommand("create database " + dbName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java
new file mode 100755
index 0000000..3ea2d9c
--- /dev/null
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLAndKerberosHiveHookIT.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.hook;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.PropertiesUtil;
+import org.apache.atlas.hive.model.HiveDataTypes;
+import org.apache.atlas.security.SecurityProperties;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
+import org.codehaus.jettison.json.JSONArray;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import javax.security.auth.Subject;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.security.PrivilegedExceptionAction;
+
+import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
+
+public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
+ public static final String TEST_USER_JAAS_SECTION = "TestUser";
+ public static final String TESTUSER = "testuser";
+ public static final String TESTPASS = "testpass";
+
+ private static final String DGI_URL = "https://localhost:21443/";
+ private Driver driver;
+ private MetadataServiceClient dgiCLient;
+ private SessionState ss;
+ private TestSecureEmbeddedServer secureEmbeddedServer;
+ private Subject subject;
+ private String originalConf;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ //Set-up hive session
+ HiveConf conf = getHiveConf();
+ driver = new Driver(conf);
+ ss = new SessionState(conf, System.getProperty("user.name"));
+ ss = SessionState.start(ss);
+ SessionState.setCurrentSessionState(ss);
+
+ jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ String persistDir = null;
+ URL resource = SSLAndKerberosHiveHookIT.class.getResource("/");
+ if (resource != null) {
+ persistDir = resource.toURI().getPath();
+ }
+ // delete prior ssl-client.xml file
+ resource = SSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
+ if (resource != null) {
+ File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
+ if (sslClientFile != null && sslClientFile.exists()) {
+ sslClientFile.delete();
+ }
+ }
+ setupKDCAndPrincipals();
+ setupCredentials();
+
+ // client will actually only leverage subset of these properties
+ final PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.setProperty(TLS_ENABLED, true);
+ configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
+ configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
+ configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+ configuration.setProperty("atlas.http.authentication.type", "kerberos");
+ configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
+
+ configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
+
+ String confLocation = System.getProperty("atlas.conf");
+ URL url;
+ if (confLocation == null) {
+ url = PropertiesUtil.class.getResource("/application.properties");
+ } else {
+ url = new File(confLocation, "application.properties").toURI().toURL();
+ }
+ configuration.load(url);
+ configuration.setProperty(TLS_ENABLED, true);
+ configuration.setProperty("atlas.http.authentication.enabled", "true");
+ configuration.setProperty("atlas.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
+ configuration.setProperty("atlas.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
+ configuration.setProperty("atlas.http.authentication.kerberos.name.rules",
+ "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
+
+ configuration.save(new FileWriter(persistDir + File.separator + "application.properties"));
+
+ dgiCLient = new MetadataServiceClient(DGI_URL) {
+ @Override
+ protected PropertiesConfiguration getClientProperties() throws MetadataException {
+ return configuration;
+ }
+ };
+
+ secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
+ @Override
+ public PropertiesConfiguration getConfiguration() {
+ return configuration;
+ }
+ };
+ WebAppContext webapp = new WebAppContext();
+ webapp.setContextPath("/");
+ webapp.setWar(System.getProperty("user.dir") + getWarPath());
+ secureEmbeddedServer.getServer().setHandler(webapp);
+
+ // save original setting
+ originalConf = System.getProperty("atlas.conf");
+ System.setProperty("atlas.conf", persistDir);
+ secureEmbeddedServer.getServer().start();
+
+ subject = loginTestUser();
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ if (secureEmbeddedServer != null) {
+ secureEmbeddedServer.getServer().stop();
+ }
+
+ if (kdc != null) {
+ kdc.stop();
+ }
+
+ if (originalConf != null) {
+ System.setProperty("atlas.conf", originalConf);
+ }
+ }
+
+ protected Subject loginTestUser() throws LoginException, IOException {
+ LoginContext lc = new LoginContext(TEST_USER_JAAS_SECTION, new CallbackHandler() {
+
+ @Override
+ public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
+ for (int i = 0; i < callbacks.length; i++) {
+ if (callbacks[i] instanceof PasswordCallback) {
+ PasswordCallback passwordCallback = (PasswordCallback) callbacks[i];
+ passwordCallback.setPassword(TESTPASS.toCharArray());
+ }
+ if (callbacks[i] instanceof NameCallback) {
+ NameCallback nameCallback = (NameCallback) callbacks[i];
+ nameCallback.setName(TESTUSER);
+ }
+ }
+ }
+ });
+ // attempt authentication
+ lc.login();
+ return lc.getSubject();
+ }
+
+ private void runCommand(final String cmd) throws Exception {
+ ss.setCommandType(null);
+ Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ driver.run(cmd);
+
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testCreateDatabase() throws Exception {
+ String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
+ runCommand("create database " + dbName);
+
+ assertDatabaseIsRegistered(dbName);
+ }
+
+ private void assertDatabaseIsRegistered(String dbName) throws Exception {
+ assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
+ }
+
+ private void assertInstanceIsRegistered(final String typeName, final String colName, final String colValue) throws Exception {
+ Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
+ Assert.assertEquals(results.length(), 1);
+
+ return null;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
new file mode 100755
index 0000000..cac313c
--- /dev/null
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.hook;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
+import org.apache.atlas.hive.model.HiveDataTypes;
+import org.apache.atlas.security.SecurityProperties;
+import org.apache.atlas.web.service.SecureEmbeddedServer;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
+import org.codehaus.jettison.json.JSONArray;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+
+import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.TLS_ENABLED;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
+
+public class SSLHiveHookIT {
+ private static final String DGI_URL = "https://localhost:21443/";
+ private Driver driver;
+ private MetadataServiceClient dgiCLient;
+ private SessionState ss;
+ private Path jksPath;
+ private String providerUrl;
+ private TestSecureEmbeddedServer secureEmbeddedServer;
+
+ class TestSecureEmbeddedServer extends SecureEmbeddedServer {
+
+ public TestSecureEmbeddedServer(int port, String path) throws IOException {
+ super(port, path);
+ }
+
+ public Server getServer () { return server; }
+
+ @Override
+ public PropertiesConfiguration getConfiguration() {
+ return super.getConfiguration();
+ }
+ }
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ //Set-up hive session
+ HiveConf conf = getHiveConf();
+ driver = new Driver(conf);
+ ss = new SessionState(conf, System.getProperty("user.name"));
+ ss = SessionState.start(ss);
+ SessionState.setCurrentSessionState(ss);
+
+ jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ String persistDir = null;
+ URL resource = SSLHiveHookIT.class.getResource("/");
+ if (resource != null) {
+ persistDir = resource.toURI().getPath();
+ }
+ // delete prior ssl-client.xml file
+ resource = SSLHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
+ if (resource != null) {
+ File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
+ if (sslClientFile != null && sslClientFile.exists()) {
+ sslClientFile.delete();
+ }
+ }
+ setupCredentials();
+
+ final PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.setProperty(TLS_ENABLED, true);
+ configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
+ configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/atlas.keystore");
+ configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+ configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
+
+ configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
+
+ dgiCLient = new MetadataServiceClient(DGI_URL) {
+ @Override
+ protected PropertiesConfiguration getClientProperties() throws MetadataException {
+ return configuration;
+ }
+ };
+
+ secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
+ @Override
+ public PropertiesConfiguration getConfiguration() {
+ return configuration;
+ }
+ };
+ WebAppContext webapp = new WebAppContext();
+ webapp.setContextPath("/");
+ webapp.setWar(System.getProperty("user.dir") + getWarPath());
+ secureEmbeddedServer.getServer().setHandler(webapp);
+
+ secureEmbeddedServer.getServer().start();
+
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ if (secureEmbeddedServer != null) {
+ secureEmbeddedServer.getServer().stop();
+ }
+ }
+
+ protected void setupCredentials() throws Exception {
+ Configuration conf = new Configuration(false);
+
+ File file = new File(jksPath.toUri().getPath());
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ // create new aliases
+ try {
+
+ char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ KEYSTORE_PASSWORD_KEY, storepass);
+
+ char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ TRUSTSTORE_PASSWORD_KEY, trustpass);
+
+ char[] trustpass2 = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ "ssl.client.truststore.password", trustpass2);
+
+ char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ SERVER_CERT_PASSWORD_KEY, certpass);
+
+ // write out so that it can be found in checks
+ provider.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ protected String getWarPath() {
+ return String.format("/../../webapp/target/atlas-webapp-%s",
+ System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
+ }
+
+ private HiveConf getHiveConf() {
+ HiveConf hiveConf = new HiveConf(this.getClass());
+ hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
+ hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/atlas");
+ hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
+ hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
+ hiveConf.set("hive.hook.dgi.synchronous", "true");
+ return hiveConf;
+ }
+
+ private void runCommand(String cmd) throws Exception {
+ ss.setCommandType(null);
+ driver.run(cmd);
+ }
+
+ @Test
+ public void testCreateDatabase() throws Exception {
+ String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
+ runCommand("create database " + dbName);
+
+ assertDatabaseIsRegistered(dbName);
+ }
+
+ private void assertDatabaseIsRegistered(String dbName) throws Exception {
+ assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
+ }
+
+ private void assertInstanceIsRegistered(String typeName, String colName, String colValue) throws Exception{
+ JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
+ Assert.assertEquals(results.length(), 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/BaseSSLAndKerberosTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/BaseSSLAndKerberosTest.java b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/BaseSSLAndKerberosTest.java
deleted file mode 100644
index 75c23bc..0000000
--- a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/BaseSSLAndKerberosTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.hive.hook;
-
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
-import org.apache.hadoop.metadata.security.BaseSecurityTest;
-import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.mortbay.jetty.Server;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
-import static org.apache.hadoop.metadata.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
-import static org.apache.hadoop.metadata.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
-
-/**
- *
- */
-public class BaseSSLAndKerberosTest extends BaseSecurityTest {
- public static final String TESTUSER = "testuser";
- public static final String TESTPASS = "testpass";
- protected static final String DGI_URL = "https://localhost:21443/";
- protected Path jksPath;
- protected String providerUrl;
- protected File httpKeytabFile;
- private File userKeytabFile;
-
- class TestSecureEmbeddedServer extends SecureEmbeddedServer {
-
- public TestSecureEmbeddedServer(int port, String path) throws IOException {
- super(port, path);
- }
-
- public Server getServer() {
- return server;
- }
-
- @Override
- public PropertiesConfiguration getConfiguration() {
- return super.getConfiguration();
- }
- }
-
- protected void setupCredentials() throws Exception {
- Configuration conf = new Configuration(false);
-
- File file = new File(jksPath.toUri().getPath());
- file.delete();
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- // create new aliases
- try {
-
- char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- KEYSTORE_PASSWORD_KEY, storepass);
-
- char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- TRUSTSTORE_PASSWORD_KEY, trustpass);
-
- char[] trustpass2 = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- "ssl.client.truststore.password", trustpass2);
-
- char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- SERVER_CERT_PASSWORD_KEY, certpass);
-
- // write out so that it can be found in checks
- provider.flush();
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- public void setupKDCAndPrincipals() throws Exception {
- // set up the KDC
- File kdcWorkDir = startKDC();
-
- userKeytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
- httpKeytabFile = createKeytab(kdc, kdcWorkDir, "HTTP", "spnego.service.keytab");
-
- // create a test user principal
- kdc.createPrincipal(TESTUSER, TESTPASS);
-
- StringBuilder jaas = new StringBuilder(1024);
- jaas.append("TestUser {\n" +
- " com.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true;\n" +
- "};\n");
- jaas.append(createJAASEntry("Client", "dgi", userKeytabFile));
- jaas.append(createJAASEntry("Server", "HTTP", httpKeytabFile));
-
- File jaasFile = new File(kdcWorkDir, "jaas.txt");
- FileUtils.write(jaasFile, jaas.toString());
- bindJVMtoJAASFile(jaasFile);
- }
-
- protected String getWarPath() {
- return String.format("/../../webapp/target/metadata-webapp-%s",
- System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
- }
-
- protected HiveConf getHiveConf() {
- HiveConf hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
- hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore");
- hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
- hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
- hiveConf.set("hive.hook.dgi.synchronous", "true");
- return hiveConf;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
deleted file mode 100755
index 33ebfde..0000000
--- a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/HiveHookIT.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.hook;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.hive.bridge.HiveMetaStoreBridge;
-import org.apache.hadoop.metadata.hive.model.HiveDataModelGenerator;
-import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.Map;
-
-public class HiveHookIT {
- public static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HiveHookIT.class);
-
- private static final String DGI_URL = "http://localhost:21000/";
- private static final String CLUSTER_NAME = "test";
- public static final String DEFAULT_DB = "default";
- private Driver driver;
- private MetadataServiceClient dgiCLient;
- private SessionState ss;
-
- @BeforeClass
- public void setUp() throws Exception {
- //Set-up hive session
- HiveConf conf = getHiveConf();
- driver = new Driver(conf);
- ss = new SessionState(conf, System.getProperty("user.name"));
- ss = SessionState.start(ss);
- SessionState.setCurrentSessionState(ss);
-
- dgiCLient = new MetadataServiceClient(DGI_URL);
- }
-
- private HiveConf getHiveConf() {
- HiveConf hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
- hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore");
- hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
- hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
- hiveConf.set("hive.hook.dgi.synchronous", "true");
- hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME);
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODE, true); //to not use hdfs
- hiveConf.setVar(HiveConf.ConfVars.HIVETESTMODEPREFIX, "");
- hiveConf.set("fs.pfile.impl", "org.apache.hadoop.fs.ProxyLocalFileSystem");
- return hiveConf;
- }
-
- private void runCommand(String cmd) throws Exception {
- ss.setCommandType(null);
- driver.run(cmd);
- }
-
- @Test
- public void testCreateDatabase() throws Exception {
- String dbName = "db" + random();
- runCommand("create database " + dbName + " WITH DBPROPERTIES ('p1'='v1', 'p2'='v2')");
- String dbId = assertDatabaseIsRegistered(dbName);
- Referenceable definition = dgiCLient.getEntity(dbId);
- Map params = (Map) definition.get("parameters");
- Assert.assertNotNull(params);
- Assert.assertEquals(params.size(), 2);
- Assert.assertEquals(params.get("p1"), "v1");
-
- //There should be just one entity per dbname
- runCommand("drop database " + dbName);
- runCommand("create database " + dbName);
- assertDatabaseIsRegistered(dbName);
- }
-
- private String dbName() {
- return "db" + random();
- }
-
- private String createDatabase() throws Exception {
- String dbName = dbName();
- runCommand("create database " + dbName);
- return dbName;
- }
-
- private String tableName() {
- return "table" + random();
- }
-
- private String createTable() throws Exception {
- return createTable(true);
- }
-
- private String createTable(boolean partition) throws Exception {
- String tableName = tableName();
- runCommand("create table " + tableName + "(id int, name string) comment 'table comment' "
- + (partition ? " partitioned by(dt string)" : ""));
- return tableName;
- }
-
- @Test
- public void testCreateTable() throws Exception {
- String tableName = tableName();
- String dbName = createDatabase();
- String colName = "col" + random();
- runCommand("create table " + dbName + "." + tableName + "(" + colName + " int, name string)");
- assertTableIsRegistered(dbName, tableName);
- //there is only one instance of column registered
- assertColumnIsRegistered(colName);
-
- tableName = createTable();
- String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
- Referenceable tableRef = dgiCLient.getEntity(tableId);
- Assert.assertEquals(tableRef.get("tableType"), TableType.MANAGED_TABLE.name());
- Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment");
- String entityName = HiveMetaStoreBridge.getTableName(CLUSTER_NAME, DEFAULT_DB, tableName);
- Assert.assertEquals(tableRef.get(HiveDataModelGenerator.NAME), entityName);
-
- final Id sdId = (Id) tableRef.get("sd");
- Referenceable sdRef = dgiCLient.getEntity(sdId.id);
- Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS),false);
-
- //Create table where database doesn't exist, will create database instance as well
- assertDatabaseIsRegistered(DEFAULT_DB);
- }
-
- private String assertColumnIsRegistered(String colName) throws Exception {
- LOG.debug("Searching for column {}", colName);
- String query = String.format("%s where name = '%s'", HiveDataTypes.HIVE_COLUMN.getName(), colName.toLowerCase());
- return assertEntityIsRegistered(query, true);
-
- }
-
- @Test
- public void testCTAS() throws Exception {
- String tableName = createTable();
- String ctasTableName = "table" + random();
- String query = "create table " + ctasTableName + " as select * from " + tableName;
- runCommand(query);
-
- assertTableIsRegistered(DEFAULT_DB, ctasTableName);
- assertProcessIsRegistered(query);
- }
-
- @Test
- public void testCreateView() throws Exception {
- String tableName = createTable();
- String viewName = tableName();
- String query = "create view " + viewName + " as select * from " + tableName;
- runCommand(query);
-
- assertTableIsRegistered(DEFAULT_DB, viewName);
- assertProcessIsRegistered(query);
- }
-
- @Test
- public void testLoadData() throws Exception {
- String tableName = createTable(false);
-
- String loadFile = file("load");
- String query = "load data local inpath 'file://" + loadFile + "' into table " + tableName;
- runCommand(query);
-
- assertProcessIsRegistered(query);
- }
-
- @Test
- public void testInsert() throws Exception {
- String tableName = createTable();
- String insertTableName = createTable();
- String query = "insert into " + insertTableName + " partition(dt = '2015-01-01') select id, name from "
- + tableName + " where dt = '2015-01-01'";
-
- runCommand(query);
- assertProcessIsRegistered(query);
- assertPartitionIsRegistered(DEFAULT_DB, insertTableName, "2015-01-01");
- }
-
- private String random() {
- return RandomStringUtils.randomAlphanumeric(10);
- }
-
- private String file(String tag) throws Exception {
- String filename = "./target/" + tag + "-data-" + random();
- File file = new File(filename);
- file.createNewFile();
- return file.getAbsolutePath();
- }
-
- private String mkdir(String tag) throws Exception {
- String filename = "./target/" + tag + "-data-" + random();
- File file = new File(filename);
- file.mkdirs();
- return file.getAbsolutePath();
- }
-
- @Test
- public void testExportImport() throws Exception {
- String tableName = createTable(false);
-
- String filename = "pfile://" + mkdir("export");
- String query = "export table " + tableName + " to \"" + filename + "\"";
- runCommand(query);
- assertProcessIsRegistered(query);
-
- tableName = createTable(false);
-
- query = "import table " + tableName + " from '" + filename + "'";
- runCommand(query);
- assertProcessIsRegistered(query);
- }
-
- @Test
- public void testSelect() throws Exception {
- String tableName = createTable();
- String query = "select * from " + tableName;
- runCommand(query);
- assertProcessIsRegistered(query);
-
- //single entity per query
- query = "SELECT * from " + tableName.toUpperCase();
- runCommand(query);
- assertProcessIsRegistered(query);
- }
-
- @Test
- public void testAlterTable() throws Exception {
- String tableName = createTable();
- String newName = tableName();
- String query = "alter table " + tableName + " rename to " + newName;
- runCommand(query);
-
- assertTableIsRegistered(DEFAULT_DB, newName);
- assertTableIsNotRegistered(DEFAULT_DB, tableName);
- }
-
- @Test
- public void testAlterView() throws Exception {
- String tableName = createTable();
- String viewName = tableName();
- String newName = tableName();
- String query = "create view " + viewName + " as select * from " + tableName;
- runCommand(query);
-
- query = "alter view " + viewName + " rename to " + newName;
- runCommand(query);
-
- assertTableIsRegistered(DEFAULT_DB, newName);
- assertTableIsNotRegistered(DEFAULT_DB, viewName);
- }
-
- private void assertProcessIsRegistered(String queryStr) throws Exception {
-// String dslQuery = String.format("%s where queryText = \"%s\"", HiveDataTypes.HIVE_PROCESS.getName(),
-// normalize(queryStr));
-// assertEntityIsRegistered(dslQuery, true);
- //todo replace with DSL
- String typeName = HiveDataTypes.HIVE_PROCESS.getName();
- String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.queryText', \"%s\").toList()",
- typeName, typeName, normalize(queryStr));
- JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
- JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertEquals(results.length(), 1);
- }
-
- private String normalize(String str) {
- if (StringUtils.isEmpty(str)) {
- return null;
- }
- return StringEscapeUtils.escapeJava(str.toLowerCase());
- }
-
- private String assertTableIsRegistered(String dbName, String tableName) throws Exception {
- return assertTableIsRegistered(dbName, tableName, true);
- }
-
- private String assertTableIsNotRegistered(String dbName, String tableName) throws Exception {
- return assertTableIsRegistered(dbName, tableName, false);
- }
-
- private String assertTableIsRegistered(String dbName, String tableName, boolean registered) throws Exception {
- LOG.debug("Searching for table {}.{}", dbName, tableName);
- String query = String.format("%s as t where tableName = '%s', db where name = '%s' and clusterName = '%s'"
- + " select t", HiveDataTypes.HIVE_TABLE.getName(), tableName.toLowerCase(), dbName.toLowerCase(),
- CLUSTER_NAME);
- return assertEntityIsRegistered(query, registered);
- }
-
- private String assertDatabaseIsRegistered(String dbName) throws Exception {
- LOG.debug("Searching for database {}", dbName);
- String query = String.format("%s where name = '%s' and clusterName = '%s'", HiveDataTypes.HIVE_DB.getName(),
- dbName.toLowerCase(), CLUSTER_NAME);
- return assertEntityIsRegistered(query, true);
- }
-
- private void assertPartitionIsRegistered(String dbName, String tableName, String value) throws Exception {
- String typeName = HiveDataTypes.HIVE_PARTITION.getName();
- String dbType = HiveDataTypes.HIVE_DB.getName();
- String tableType = HiveDataTypes.HIVE_TABLE.getName();
-
- LOG.debug("Searching for partition of {}.{} with values {}", dbName, tableName, value);
- //todo replace with DSL
- String gremlinQuery = String.format("g.V.has('__typeName', '%s').has('%s.values', ['%s']).as('p')."
- + "out('__%s.table').has('%s.tableName', '%s').out('__%s.db').has('%s.name', '%s')"
- + ".has('%s.clusterName', '%s').back('p').toList()", typeName, typeName, value, typeName,
- tableType, tableName.toLowerCase(), tableType, dbType, dbName.toLowerCase(), dbType, CLUSTER_NAME);
- JSONObject response = dgiCLient.searchByGremlin(gremlinQuery);
- JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertEquals(results.length(), 1);
- }
-
- private String assertEntityIsRegistered(String dslQuery, boolean registered) throws Exception{
- JSONArray results = dgiCLient.searchByDSL(dslQuery);
- if (registered) {
- Assert.assertEquals(results.length(), 1);
- JSONObject row = results.getJSONObject(0);
- if (row.has("$id$")) {
- return row.getJSONObject("$id$").getString("id");
- } else {
- return row.getJSONObject("_col_0").getString("id");
- }
- } else {
- Assert.assertEquals(results.length(), 0);
- return null;
- }
- }
-
- @Test
- public void testLineage() throws Exception {
- String table1 = createTable(false);
-
- String db2 = createDatabase();
- String table2 = tableName();
-
- String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
- runCommand(query);
- String table1Id = assertTableIsRegistered(DEFAULT_DB, table1);
- String table2Id = assertTableIsRegistered(db2, table2);
-
- String datasetName = HiveMetaStoreBridge.getTableName(CLUSTER_NAME, db2, table2);
- JSONObject response = dgiCLient.getInputGraph(datasetName);
- JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
- Assert.assertTrue(vertices.has(table1Id));
- Assert.assertTrue(vertices.has(table2Id));
-
- datasetName = HiveMetaStoreBridge.getTableName(CLUSTER_NAME, DEFAULT_DB, table1);
- response = dgiCLient.getOutputGraph(datasetName);
- vertices = response.getJSONObject("values").getJSONObject("vertices");
- Assert.assertTrue(vertices.has(table1Id));
- Assert.assertTrue(vertices.has(table2Id));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/NegativeSSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
deleted file mode 100755
index af073f5..0000000
--- a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/NegativeSSLAndKerberosHiveHookIT.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.hook;
-
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.apache.hadoop.metadata.security.SecurityProperties;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
-import org.mortbay.jetty.webapp.WebAppContext;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.net.URL;
-import java.nio.file.Files;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-/**
- * Perform all the necessary setup steps for client and server comm over SSL/Kerberos, but then don't estalish a
- * kerberos user for the invocation. Need a separate use case since the Jersey layer cached the URL connection handler,
- * which indirectly caches the kerberos delegation token.
- */
-public class NegativeSSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
-
- private Driver driver;
- private SessionState ss;
- private TestSecureEmbeddedServer secureEmbeddedServer;
- private String originalConf;
-
- @BeforeClass
- public void setUp() throws Exception {
- //Set-up hive session
- HiveConf conf = getHiveConf();
- driver = new Driver(conf);
- ss = new SessionState(conf, System.getProperty("user.name"));
- ss = SessionState.start(ss);
- SessionState.setCurrentSessionState(ss);
-
- jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
-
- String persistDir = null;
- URL resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/");
- if (resource != null) {
- persistDir = resource.toURI().getPath();
- }
- // delete prior ssl-client.xml file
- resource = NegativeSSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
- if (resource != null) {
- File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
- if (sslClientFile != null && sslClientFile.exists()) {
- sslClientFile.delete();
- }
- }
- setupKDCAndPrincipals();
- setupCredentials();
-
- // client will actually only leverage subset of these properties
- final PropertiesConfiguration configuration = new PropertiesConfiguration();
- configuration.setProperty(TLS_ENABLED, true);
- configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/metadata.keystore");
- configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/metadata.keystore");
- configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
- configuration.setProperty("metadata.http.authentication.type", "kerberos");
- configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
-
- configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
-
- String confLocation = System.getProperty("metadata.conf");
- URL url;
- if (confLocation == null) {
- url = PropertiesUtil.class.getResource("/application.properties");
- } else {
- url = new File(confLocation, "application.properties").toURI().toURL();
- }
- configuration.load(url);
- configuration.setProperty(TLS_ENABLED, true);
- configuration.setProperty("metadata.http.authentication.enabled", "true");
- configuration.setProperty("metadata.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
- configuration.setProperty("metadata.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
- configuration.setProperty("metadata.http.authentication.kerberos.name.rules",
- "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
-
- configuration.save(new FileWriter(persistDir + File.separator + "application.properties"));
-
- secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
- @Override
- public PropertiesConfiguration getConfiguration() {
- return configuration;
- }
- };
- WebAppContext webapp = new WebAppContext();
- webapp.setContextPath("/");
- webapp.setWar(System.getProperty("user.dir") + getWarPath());
- secureEmbeddedServer.getServer().setHandler(webapp);
-
- // save original setting
- originalConf = System.getProperty("metadata.conf");
- System.setProperty("metadata.conf", persistDir);
- secureEmbeddedServer.getServer().start();
-
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- if (secureEmbeddedServer != null) {
- secureEmbeddedServer.getServer().stop();
- }
-
- if (kdc != null) {
- kdc.stop();
- }
-
- if (originalConf != null) {
- System.setProperty("metadata.conf", originalConf);
- }
- }
-
- private void runCommand(final String cmd) throws Exception {
- ss.setCommandType(null);
- driver.run(cmd);
- Assert.assertNotNull(driver.getErrorMsg());
- Assert.assertTrue(driver.getErrorMsg().contains("Mechanism level: Failed to find any Kerberos tgt"));
- }
-
- @Test
- public void testUnsecuredCreateDatabase() throws Exception {
- String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
- runCommand("create database " + dbName);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLAndKerberosHiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLAndKerberosHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLAndKerberosHiveHookIT.java
deleted file mode 100755
index 62c8a69..0000000
--- a/addons/hive-bridge/src/test/java/org/apache/hadoop/metadata/hive/hook/SSLAndKerberosHiveHookIT.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.hive.hook;
-
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.PropertiesUtil;
-import org.apache.hadoop.metadata.hive.model.HiveDataTypes;
-import org.apache.hadoop.metadata.security.SecurityProperties;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.apache.hadoop.security.ssl.SSLHostnameVerifier;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.mortbay.jetty.webapp.WebAppContext;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.security.auth.Subject;
-import javax.security.auth.callback.*;
-import javax.security.auth.login.LoginContext;
-import javax.security.auth.login.LoginException;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.security.PrivilegedExceptionAction;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-public class SSLAndKerberosHiveHookIT extends BaseSSLAndKerberosTest {
- public static final String TEST_USER_JAAS_SECTION = "TestUser";
- public static final String TESTUSER = "testuser";
- public static final String TESTPASS = "testpass";
-
- private static final String DGI_URL = "https://localhost:21443/";
- private Driver driver;
- private MetadataServiceClient dgiCLient;
- private SessionState ss;
- private TestSecureEmbeddedServer secureEmbeddedServer;
- private Subject subject;
- private String originalConf;
-
- @BeforeClass
- public void setUp() throws Exception {
- //Set-up hive session
- HiveConf conf = getHiveConf();
- driver = new Driver(conf);
- ss = new SessionState(conf, System.getProperty("user.name"));
- ss = SessionState.start(ss);
- SessionState.setCurrentSessionState(ss);
-
- jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
-
- String persistDir = null;
- URL resource = SSLAndKerberosHiveHookIT.class.getResource("/");
- if (resource != null) {
- persistDir = resource.toURI().getPath();
- }
- // delete prior ssl-client.xml file
- resource = SSLAndKerberosHiveHookIT.class.getResource("/" + SecurityProperties.SSL_CLIENT_PROPERTIES);
- if (resource != null) {
- File sslClientFile = new File(persistDir, SecurityProperties.SSL_CLIENT_PROPERTIES);
- if (sslClientFile != null && sslClientFile.exists()) {
- sslClientFile.delete();
- }
- }
- setupKDCAndPrincipals();
- setupCredentials();
-
- // client will actually only leverage subset of these properties
- final PropertiesConfiguration configuration = new PropertiesConfiguration();
- configuration.setProperty(TLS_ENABLED, true);
- configuration.setProperty(TRUSTSTORE_FILE_KEY, "../../webapp/target/metadata.keystore");
- configuration.setProperty(KEYSTORE_FILE_KEY, "../../webapp/target/metadata.keystore");
- configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
- configuration.setProperty("metadata.http.authentication.type", "kerberos");
- configuration.setProperty(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, SSLHostnameVerifier.DEFAULT_AND_LOCALHOST.toString());
-
- configuration.save(new FileWriter(persistDir + File.separator + "client.properties"));
-
- String confLocation = System.getProperty("metadata.conf");
- URL url;
- if (confLocation == null) {
- url = PropertiesUtil.class.getResource("/application.properties");
- } else {
- url = new File(confLocation, "application.properties").toURI().toURL();
- }
- configuration.load(url);
- configuration.setProperty(TLS_ENABLED, true);
- configuration.setProperty("metadata.http.authentication.enabled", "true");
- configuration.setProperty("metadata.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
- configuration.setProperty("metadata.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
- configuration.setProperty("metadata.http.authentication.kerberos.name.rules",
- "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
-
- configuration.save(new FileWriter(persistDir + File.separator + "application.properties"));
-
- dgiCLient = new MetadataServiceClient(DGI_URL) {
- @Override
- protected PropertiesConfiguration getClientProperties() throws MetadataException {
- return configuration;
- }
- };
-
- secureEmbeddedServer = new TestSecureEmbeddedServer(21443, "webapp/target/metadata-governance") {
- @Override
- public PropertiesConfiguration getConfiguration() {
- return configuration;
- }
- };
- WebAppContext webapp = new WebAppContext();
- webapp.setContextPath("/");
- webapp.setWar(System.getProperty("user.dir") + getWarPath());
- secureEmbeddedServer.getServer().setHandler(webapp);
-
- // save original setting
- originalConf = System.getProperty("metadata.conf");
- System.setProperty("metadata.conf", persistDir);
- secureEmbeddedServer.getServer().start();
-
- subject = loginTestUser();
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- if (secureEmbeddedServer != null) {
- secureEmbeddedServer.getServer().stop();
- }
-
- if (kdc != null) {
- kdc.stop();
- }
-
- if (originalConf != null) {
- System.setProperty("metadata.conf", originalConf);
- }
- }
-
- protected Subject loginTestUser() throws LoginException, IOException {
- LoginContext lc = new LoginContext(TEST_USER_JAAS_SECTION, new CallbackHandler() {
-
- @Override
- public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
- for (int i = 0; i < callbacks.length; i++) {
- if (callbacks[i] instanceof PasswordCallback) {
- PasswordCallback passwordCallback = (PasswordCallback) callbacks[i];
- passwordCallback.setPassword(TESTPASS.toCharArray());
- }
- if (callbacks[i] instanceof NameCallback) {
- NameCallback nameCallback = (NameCallback) callbacks[i];
- nameCallback.setName(TESTUSER);
- }
- }
- }
- });
- // attempt authentication
- lc.login();
- return lc.getSubject();
- }
-
- private void runCommand(final String cmd) throws Exception {
- ss.setCommandType(null);
- Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- driver.run(cmd);
-
- return null;
- }
- });
- }
-
- @Test
- public void testCreateDatabase() throws Exception {
- String dbName = "db" + RandomStringUtils.randomAlphanumeric(5).toLowerCase();
- runCommand("create database " + dbName);
-
- assertDatabaseIsRegistered(dbName);
- }
-
- private void assertDatabaseIsRegistered(String dbName) throws Exception {
- assertInstanceIsRegistered(HiveDataTypes.HIVE_DB.getName(), "name", dbName);
- }
-
- private void assertInstanceIsRegistered(final String typeName, final String colName, final String colValue) throws Exception {
- Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- JSONArray results = dgiCLient.rawSearch(typeName, colName, colValue);
- Assert.assertEquals(results.length(), 1);
-
- return null;
- }
- });
- }
-}
[03/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
new file mode 100755
index 0000000..be6ae8b
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
@@ -0,0 +1,679 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.google.common.collect.ImmutableList;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.MetadataServiceException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.InstanceSerialization$;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization$;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.atlas.web.util.Servlets;
+import org.apache.commons.lang.RandomStringUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * Integration tests for Entity Jersey Resource.
+ */
+public class EntityJerseyResourceIT extends BaseResourceIT {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class);
+
+ private static final String DATABASE_TYPE = "hive_database";
+ private static final String DATABASE_NAME = "foo";
+ private static final String TABLE_TYPE = "hive_table_type";
+ private static final String TABLE_NAME = "bar";
+ private static final String TRAITS = "traits";
+
+ private Referenceable tableInstance;
+ private Id tableId;
+ private String traitName;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ super.setUp();
+
+ createHiveTypes();
+ }
+
+ @Test
+ public void testSubmitEntity() throws Exception {
+ tableInstance = createHiveTableInstance();
+ tableId = createInstance(tableInstance);
+
+ final String guid = tableId._getId();
+ try {
+ Assert.assertNotNull(UUID.fromString(guid));
+ } catch (IllegalArgumentException e) {
+ Assert.fail("Response is not a guid, " + guid);
+ }
+ }
+
+ @DataProvider
+ public Object[][] invalidAttrValues() {
+ return new Object[][]{
+ {null}, {""}, {" "}};
+ }
+
+ @Test(dataProvider = "invalidAttrValues")
+ public void testEntityInvalidValue(String value) throws Exception {
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ databaseInstance.set("name", randomString());
+ databaseInstance.set("description", value);
+
+ try {
+ createInstance(databaseInstance);
+ Assert.fail("Exptected MetadataServiceException");
+ } catch(MetadataServiceException e) {
+ Assert.assertEquals(e.getStatus(), ClientResponse.Status.BAD_REQUEST);
+ }
+ }
+
+ @Test
+ public void testSubmitEntityWithBadDateFormat() throws Exception {
+
+ try {
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ databaseInstance.set("name", DATABASE_NAME);
+ databaseInstance.set("description", "foo database");
+
+ Referenceable tableInstance = new Referenceable(TABLE_TYPE,
+ "classification", "pii", "phi", "pci", "sox", "sec", "finance");
+ tableInstance.set("name", TABLE_NAME);
+ tableInstance.set("description", "bar table");
+ tableInstance.set("date", "2014-07-11");
+ tableInstance.set("type", "managed");
+ tableInstance.set("level", 2);
+ tableInstance.set("tableType", 1); // enum
+ tableInstance.set("database", databaseInstance);
+ tableInstance.set("compressed", false);
+
+ Struct traitInstance = (Struct) tableInstance.getTrait("classification");
+ traitInstance.set("tag", "foundation_etl");
+
+ Struct serde1Instance = new Struct("serdeType");
+ serde1Instance.set("name", "serde1");
+ serde1Instance.set("serde", "serde1");
+ tableInstance.set("serde1", serde1Instance);
+
+ Struct serde2Instance = new Struct("serdeType");
+ serde2Instance.set("name", "serde2");
+ serde2Instance.set("serde", "serde2");
+ tableInstance.set("serde2", serde2Instance);
+
+ tableId = createInstance(tableInstance);
+ Assert.fail("Was expecting an exception here ");
+ } catch (MetadataServiceException e) {
+ Assert.assertTrue(
+ e.getMessage().contains("\"error\":\"Cannot convert value '2014-07-11' to datatype date\""));
+ }
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testAddProperty() throws Exception {
+ final String guid = tableId._getId();
+ //add property
+ String description = "bar table - new desc";
+ ClientResponse clientResponse = addProperty(guid, "description", description);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String entityRef = getEntityDefinition(getEntityDefinition(guid));
+ Assert.assertNotNull(entityRef);
+
+ tableInstance.set("description", description);
+
+ //invalid property for the type
+ clientResponse = addProperty(guid, "invalid_property", "bar table");
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+
+ //non-string property, update
+ clientResponse = addProperty(guid, "level", "4");
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ entityRef = getEntityDefinition(getEntityDefinition(guid));
+ Assert.assertNotNull(entityRef);
+
+ tableInstance.set("level", 4);
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
+ public void testAddNullProperty() throws Exception {
+ final String guid = tableId._getId();
+ //add property
+ addProperty(guid, null, "foo bar");
+ Assert.fail();
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
+ public void testAddNullPropertyValue() throws Exception {
+ final String guid = tableId._getId();
+ //add property
+ addProperty(guid, "description", null);
+ Assert.fail();
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testAddReferenceProperty() throws Exception {
+ //Create new db instance
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ databaseInstance.set("name", "newdb");
+ databaseInstance.set("description", "new database");
+
+ Id dbInstance = createInstance(databaseInstance);
+ String dbId = dbInstance._getId();
+
+ //Add reference property
+ final String guid = tableId._getId();
+ ClientResponse clientResponse = addProperty(guid, "database", dbId);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testGetEntityDefinition() throws Exception {
+ final String guid = tableId._getId();
+ ClientResponse clientResponse = getEntityDefinition(guid);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ final String definition = response.getString(MetadataServiceClient.DEFINITION);
+ Assert.assertNotNull(definition);
+ LOG.debug("tableInstanceAfterGet = " + definition);
+ InstanceSerialization.fromJsonReferenceable(definition, true);
+ }
+
+ private ClientResponse addProperty(String guid, String property, String value) {
+ WebResource resource = service
+ .path("api/atlas/entities")
+ .path(guid);
+
+ return resource.queryParam("property", property).queryParam("value", value)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.PUT, ClientResponse.class);
+ }
+
+ private ClientResponse getEntityDefinition(String guid) {
+ WebResource resource = service
+ .path("api/atlas/entities")
+ .path(guid);
+ return resource.accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ }
+
+ private String getEntityDefinition(ClientResponse clientResponse) throws Exception {
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+ JSONObject response = new JSONObject(clientResponse.getEntity(String.class));
+ final String definition = response.getString(MetadataServiceClient.DEFINITION);
+ Assert.assertNotNull(definition);
+
+ return definition;
+ }
+
+ @Test
+ public void testGetInvalidEntityDefinition() throws Exception {
+ WebResource resource = service
+ .path("api/atlas/entities")
+ .path("blah");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
+ Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testGetEntityList() throws Exception {
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .queryParam("type", TABLE_TYPE)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertNotNull(list);
+ Assert.assertEquals(list.length(), 1);
+ }
+
+ @Test
+ public void testGetEntityListForBadEntityType() throws Exception {
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .queryParam("type", "blah")
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
+ Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
+ }
+
+
+ @Test
+ public void testGetEntityListForNoInstances() throws Exception {
+ addNewType();
+
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .queryParam("type", "test")
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertEquals(list.length(), 0);
+ }
+
+ private void addNewType() throws Exception {
+ HierarchicalTypeDefinition<ClassType> testTypeDefinition =
+ TypesUtil.createClassTypeDef("test", ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+
+ String typesAsJSON = TypesSerialization.toJson(testTypeDefinition);
+ createType(typesAsJSON);
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testGetTraitNames() throws Exception {
+ final String guid = tableId._getId();
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .path(guid)
+ .path(TRAITS)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+ Assert.assertNotNull(response.get("GUID"));
+
+ final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertEquals(list.length(), 7);
+ }
+
+ @Test(dependsOnMethods = "testGetTraitNames")
+ public void testAddTrait() throws Exception {
+ traitName = "PII_Trait" + randomString();
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
+ String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
+ LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
+ createType(traitDefinitionAsJSON);
+
+ Struct traitInstance = new Struct(traitName);
+ String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
+ LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
+
+ final String guid = tableId._getId();
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .path(guid)
+ .path(TRAITS)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+ Assert.assertNotNull(response.get(MetadataServiceClient.GUID));
+ }
+
+ @Test(dependsOnMethods = "testAddTrait")
+ public void testAddExistingTrait() throws Exception {
+ final String traitName = "PII_Trait" + randomString();
+
+ Struct traitInstance = new Struct(traitName);
+ String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
+ LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
+
+ final String guid = tableId._getId();
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .path(guid)
+ .path(TRAITS)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testGetTraitNames")
+ public void testAddTraitWithAttribute() throws Exception {
+ final String traitName = "PII_Trait" + randomString();
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
+ String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
+ LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
+ createType(traitDefinitionAsJSON);
+
+ Struct traitInstance = new Struct(traitName);
+ traitInstance.set("type", "SSN");
+ String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
+ LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
+
+ final String guid = tableId._getId();
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .path(guid)
+ .path(TRAITS)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+ Assert.assertNotNull(response.get(MetadataServiceClient.GUID));
+
+ // verify the response
+ clientResponse = getEntityDefinition(guid);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+ responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+ response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ final String definition = response.getString(MetadataServiceClient.DEFINITION);
+ Assert.assertNotNull(definition);
+ Referenceable entityRef = InstanceSerialization.fromJsonReferenceable(definition, true);
+ IStruct traitRef = entityRef.getTrait(traitName);
+ String type = (String) traitRef.get("type");
+ Assert.assertEquals(type, "SSN");
+ }
+
+ @Test
+ public void testAddTraitWithNoRegistration() throws Exception {
+ final String traitName = "PII_Trait" + randomString();
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
+ String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
+ LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
+
+ Struct traitInstance = new Struct(traitName);
+ String traitInstanceAsJSON = InstanceSerialization$.MODULE$.toJson(traitInstance, true);
+ LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
+
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .path("random")
+ .path(TRAITS)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testAddTrait")
+ public void testDeleteTrait() throws Exception {
+ final String guid = tableId._getId();
+
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .path(guid)
+ .path(TRAITS)
+ .path(traitName)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.DELETE, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+ Assert.assertNotNull(response.get("GUID"));
+ Assert.assertNotNull(response.get("traitName"));
+ }
+
+ @Test
+ public void testDeleteTraitNonExistent() throws Exception {
+ final String traitName = "blah_trait";
+
+ ClientResponse clientResponse = service
+ .path("api/atlas/entities")
+ .path("random")
+ .path(TRAITS)
+ .path(traitName)
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.DELETE, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
+ Assert.assertEquals(response.getString(MetadataServiceClient.ERROR),
+ "trait=" + traitName + " should be defined in type system before it can be deleted");
+ Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
+ }
+
+ private String random() {
+ return RandomStringUtils.random(10);
+ }
+
+ private String randomString() {
+ return RandomStringUtils.randomAlphanumeric(10);
+ }
+
+ @Test
+ public void testUTF8() throws Exception {
+ String classType = random();
+ String attrName = random();
+ String attrValue = random();
+
+ HierarchicalTypeDefinition<ClassType> classTypeDefinition =
+ TypesUtil.createClassTypeDef(classType, ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE));
+ TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(classTypeDefinition));
+ createType(typesDef);
+
+ Referenceable instance = new Referenceable(classType);
+ instance.set(attrName, attrValue);
+ Id guid = createInstance(instance);
+
+ ClientResponse response = getEntityDefinition(guid._getId());
+ String definition = getEntityDefinition(response);
+ Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
+ Assert.assertEquals(getReferenceable.get(attrName), attrValue);
+ }
+
+ private void createHiveTypes() throws Exception {
+ HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
+ TypesUtil.createClassTypeDef(DATABASE_TYPE,
+ ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+
+ StructTypeDefinition structTypeDefinition =
+ new StructTypeDefinition("serdeType",
+ new AttributeDefinition[]{
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
+ });
+
+ EnumValue values[] = {
+ new EnumValue("MANAGED", 1),
+ new EnumValue("EXTERNAL", 2),
+ };
+
+ EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
+
+ HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
+ TypesUtil.createClassTypeDef(TABLE_TYPE,
+ ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("date", DataTypes.DATE_TYPE),
+ TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE),
+ new AttributeDefinition("tableType", "tableType",
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("serde1",
+ "serdeType", Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("serde2",
+ "serdeType", Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("database",
+ DATABASE_TYPE, Multiplicity.REQUIRED, true, null),
+ new AttributeDefinition("compressed",
+ DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, true, null));
+
+ HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
+ TypesUtil.createTraitTypeDef("classification",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef("pii", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> phiTrait =
+ TypesUtil.createTraitTypeDef("phi", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> pciTrait =
+ TypesUtil.createTraitTypeDef("pci", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> soxTrait =
+ TypesUtil.createTraitTypeDef("sox", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> secTrait =
+ TypesUtil.createTraitTypeDef("sec", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> financeTrait =
+ TypesUtil.createTraitTypeDef("finance", ImmutableList.<String>of());
+
+ TypesDef typesDef = TypeUtils.getTypesDef(
+ ImmutableList.of(enumTypeDefinition),
+ ImmutableList.of(structTypeDefinition),
+ ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
+ soxTrait, secTrait, financeTrait),
+ ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
+ createType(typesDef);
+ }
+
+ private Referenceable createHiveTableInstance() throws Exception {
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ databaseInstance.set("name", DATABASE_NAME);
+ databaseInstance.set("description", "foo database");
+
+ Referenceable tableInstance = new Referenceable(TABLE_TYPE,
+ "classification", "pii", "phi", "pci", "sox", "sec", "finance");
+ tableInstance.set("name", TABLE_NAME);
+ tableInstance.set("description", "bar table");
+ tableInstance.set("date", "2014-07-11T08:00:00.000Z");
+ tableInstance.set("type", "managed");
+ tableInstance.set("level", 2);
+ tableInstance.set("tableType", 1); // enum
+ tableInstance.set("database", databaseInstance);
+ tableInstance.set("compressed", false);
+
+ Struct traitInstance = (Struct) tableInstance.getTrait("classification");
+ traitInstance.set("tag", "foundation_etl");
+
+ Struct serde1Instance = new Struct("serdeType");
+ serde1Instance.set("name", "serde1");
+ serde1Instance.set("serde", "serde1");
+ tableInstance.set("serde1", serde1Instance);
+
+ Struct serde2Instance = new Struct("serdeType");
+ serde2Instance.set("name", "serde2");
+ serde2Instance.set("serde", "serde2");
+ tableInstance.set("serde2", serde2Instance);
+
+ List<String> traits = tableInstance.getTraits();
+ Assert.assertEquals(traits.size(), 7);
+
+ return tableInstance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java
new file mode 100644
index 0000000..421ce15
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+import java.util.List;
+
+/**
+ * Hive Lineage Integration Tests.
+ */
+public class HiveLineageJerseyResourceIT extends BaseResourceIT {
+
+ private static final String BASE_URI = "api/atlas/lineage/hive/table/";
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ super.setUp();
+
+ setUpTypes();
+ setupInstances();
+ }
+
+ @Test
+ public void testInputsGraph() throws Exception {
+ WebResource resource = service
+ .path(BASE_URI)
+ .path("sales_fact_monthly_mv")
+ .path("inputs")
+ .path("graph");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+ System.out.println("inputs graph = " + responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
+ Assert.assertNotNull(results);
+
+ JSONObject values = results.getJSONObject("values");
+ Assert.assertNotNull(values);
+
+ final JSONObject vertices = values.getJSONObject("vertices");
+ Assert.assertEquals(vertices.length(), 4);
+
+ final JSONObject edges = values.getJSONObject("edges");
+ Assert.assertEquals(edges.length(), 4);
+ }
+
+ @Test
+ public void testOutputsGraph() throws Exception {
+ WebResource resource = service
+ .path(BASE_URI)
+ .path("sales_fact")
+ .path("outputs")
+ .path("graph");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+ System.out.println("outputs graph= " + responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
+ Assert.assertNotNull(results);
+
+ JSONObject values = results.getJSONObject("values");
+ Assert.assertNotNull(values);
+
+ final JSONObject vertices = values.getJSONObject("vertices");
+ Assert.assertEquals(vertices.length(), 3);
+
+ final JSONObject edges = values.getJSONObject("edges");
+ Assert.assertEquals(edges.length(), 4);
+ }
+
+ @Test
+ public void testSchema() throws Exception {
+ WebResource resource = service
+ .path(BASE_URI)
+ .path("sales_fact")
+ .path("schema");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+ System.out.println("schema = " + responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
+ Assert.assertNotNull(results);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertEquals(rows.length(), 4);
+
+ for (int index = 0; index < rows.length(); index++) {
+ final JSONObject row = rows.getJSONObject(index);
+ Assert.assertNotNull(row.getString("name"));
+ Assert.assertNotNull(row.getString("comment"));
+ Assert.assertNotNull(row.getString("dataType"));
+ Assert.assertEquals(row.getString("$typeName$"), "hive_column");
+ }
+ }
+
+ @Test
+ public void testSchemaForEmptyTable() throws Exception {
+ WebResource resource = service
+ .path(BASE_URI)
+ .path("")
+ .path("schema");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(),
+ Response.Status.NOT_FOUND.getStatusCode());
+ }
+
+ @Test
+ public void testSchemaForInvalidTable() throws Exception {
+ WebResource resource = service
+ .path(BASE_URI)
+ .path("blah")
+ .path("schema");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(),
+ Response.Status.NOT_FOUND.getStatusCode());
+ }
+
+ private void setUpTypes() throws Exception {
+ TypesDef typesDef = createTypeDefinitions();
+ createType(typesDef);
+ }
+
+ private static final String DATABASE_TYPE = "hive_db";
+ private static final String HIVE_TABLE_TYPE = "hive_table";
+ private static final String COLUMN_TYPE = "hive_column";
+ private static final String HIVE_PROCESS_TYPE = "hive_process";
+
+ private TypesDef createTypeDefinitions() {
+ HierarchicalTypeDefinition<ClassType> dbClsDef
+ = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("description", DataTypes.STRING_TYPE),
+ attrDef("locationUri", DataTypes.STRING_TYPE),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.INT_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> columnClsDef =
+ TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("dataType", DataTypes.STRING_TYPE),
+ attrDef("comment", DataTypes.STRING_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> tblClsDef =
+ TypesUtil.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.INT_TYPE),
+ attrDef("lastAccessTime", DataTypes.INT_TYPE),
+ attrDef("tableType", DataTypes.STRING_TYPE),
+ attrDef("temporary", DataTypes.BOOLEAN_TYPE),
+ new AttributeDefinition("db", DATABASE_TYPE,
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("columns",
+ DataTypes.arrayTypeName(COLUMN_TYPE),
+ Multiplicity.COLLECTION, true, null)
+ );
+
+ HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
+ TypesUtil.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
+ attrDef("userName", DataTypes.STRING_TYPE),
+ attrDef("startTime", DataTypes.INT_TYPE),
+ attrDef("endTime", DataTypes.INT_TYPE),
+ attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
+ );
+
+ HierarchicalTypeDefinition<TraitType> dimTraitDef =
+ TypesUtil.createTraitTypeDef("Dimension", null);
+
+ HierarchicalTypeDefinition<TraitType> factTraitDef =
+ TypesUtil.createTraitTypeDef("Fact", null);
+
+ HierarchicalTypeDefinition<TraitType> metricTraitDef =
+ TypesUtil.createTraitTypeDef("Metric", null);
+
+ HierarchicalTypeDefinition<TraitType> etlTraitDef =
+ TypesUtil.createTraitTypeDef("ETL", null);
+
+
+ HierarchicalTypeDefinition<TraitType> piiTraitDef =
+ TypesUtil.createTraitTypeDef("PII", null);
+
+ return TypeUtils.getTypesDef(
+ ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef, piiTraitDef),
+ ImmutableList.of(dbClsDef, columnClsDef, tblClsDef, loadProcessClsDef)
+ );
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT) {
+ return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
+ return attrDef(name, dT, m, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT,
+ Multiplicity m, boolean isComposite, String reverseAttributeName) {
+ Preconditions.checkNotNull(name);
+ Preconditions.checkNotNull(dT);
+ return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
+ }
+
+ private void setupInstances() throws Exception {
+ Id salesDB = database(
+ "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
+
+ List<Referenceable> salesFactColumns = ImmutableList.of(
+ column("time_id", "int", "time id"),
+ column("product_id", "int", "product id"),
+ column("customer_id", "int", "customer id", "PII"),
+ column("sales", "double", "product id", "Metric")
+ );
+
+ Id salesFact = table("sales_fact", "sales fact table",
+ salesDB, "Joe", "Managed", salesFactColumns, "Fact");
+
+ List<Referenceable> timeDimColumns = ImmutableList.of(
+ column("time_id", "int", "time id"),
+ column("dayOfYear", "int", "day Of Year"),
+ column("weekDay", "int", "week Day")
+ );
+
+ Id timeDim = table("time_dim", "time dimension table",
+ salesDB, "John Doe", "External", timeDimColumns, "Dimension");
+
+ Id reportingDB = database("Reporting", "reporting database", "Jane BI",
+ "hdfs://host:8000/apps/warehouse/reporting");
+
+ Id salesFactDaily = table("sales_fact_daily_mv",
+ "sales fact daily materialized view",
+ reportingDB, "Joe BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesDaily", "John ETL",
+ ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+
+ Id salesFactMonthly = table("sales_fact_monthly_mv",
+ "sales fact monthly materialized view",
+ reportingDB, "Jane BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesMonthly", "John ETL",
+ ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+ }
+
+ Id database(String name, String description,
+ String owner, String locationUri,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("locationUri", locationUri);
+ referenceable.set("createTime", System.currentTimeMillis());
+
+ return createInstance(referenceable);
+ }
+
+ Referenceable column(String name, String dataType, String comment,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("dataType", dataType);
+ referenceable.set("comment", comment);
+
+ return referenceable;
+ }
+
+ Id table(String name, String description, Id dbId,
+ String owner, String tableType,
+ List<Referenceable> columns,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("tableType", tableType);
+ referenceable.set("createTime", System.currentTimeMillis());
+ referenceable.set("lastAccessTime", System.currentTimeMillis());
+ referenceable.set("retention", System.currentTimeMillis());
+
+ referenceable.set("db", dbId);
+ referenceable.set("columns", columns);
+
+ return createInstance(referenceable);
+ }
+
+ Id loadProcess(String name, String user,
+ List<Id> inputTables,
+ List<Id> outputTables,
+ String queryText, String queryPlan,
+ String queryId, String queryGraph,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("user", user);
+ referenceable.set("startTime", System.currentTimeMillis());
+ referenceable.set("endTime", System.currentTimeMillis() + 10000);
+
+ referenceable.set("inputs", inputTables);
+ referenceable.set("outputs", outputTables);
+
+ referenceable.set("queryText", queryText);
+ referenceable.set("queryPlan", queryPlan);
+ referenceable.set("queryId", queryId);
+ referenceable.set("queryGraph", queryGraph);
+
+ return createInstance(referenceable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
new file mode 100755
index 0000000..cf27ac2
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.google.common.collect.ImmutableList;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+import java.util.List;
+
+/**
+ * Search Integration Tests.
+ */
+public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ super.setUp();
+
+ createTypes();
+ createInstance();
+ }
+
+ @Test
+ public void testSearchByDSL() throws Exception {
+ String dslQuery = "from dsl_test_type";
+ WebResource resource = service
+ .path("api/atlas/discovery/search/dsl")
+ .queryParam("query", dslQuery);
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ Assert.assertEquals(response.getString("query"), dslQuery);
+ Assert.assertEquals(response.getString("queryType"), "dsl");
+
+ JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
+ Assert.assertNotNull(results);
+
+ JSONArray rows = results.getJSONArray(MetadataServiceClient.ROWS);
+ Assert.assertEquals(rows.length(), 1);
+
+ int numRows = response.getInt(MetadataServiceClient.COUNT);
+ Assert.assertEquals(numRows, 1);
+
+ }
+
+ @Test
+ public void testSearchByDSLForUnknownType() throws Exception {
+ String dslQuery = "from blah";
+ WebResource resource = service
+ .path("api/atlas/discovery/search/dsl")
+ .queryParam("query", dslQuery);
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(),
+ Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test
+ public void testSearchUsingGremlin() throws Exception {
+ String query = "g.V.has('type', 'dsl_test_type').toList()";
+ WebResource resource = service
+ .path("api/atlas/discovery/search")
+ .queryParam("query", query);
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ Assert.assertEquals(response.getString("query"), query);
+ Assert.assertEquals(response.getString("queryType"), "gremlin");
+ }
+
+ @Test
+ public void testSearchUsingDSL() throws Exception {
+ String query = "from dsl_test_type";
+ WebResource resource = service
+ .path("api/atlas/discovery/search")
+ .queryParam("query", query);
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ Assert.assertEquals(response.getString("query"), query);
+ Assert.assertEquals(response.getString("queryType"), "dsl");
+ }
+
+ @Test(enabled = false)
+ public void testSearchUsingFullText() throws Exception {
+ String query = "foundation_etl";
+ JSONObject response = serviceClient.searchByFullText(query);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ Assert.assertEquals(response.getString("query"), query);
+ Assert.assertEquals(response.getString("queryType"), "full-text");
+
+ JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertEquals(results.length(), 1);
+
+ JSONObject row = results.getJSONObject(0);
+ Assert.assertNotNull(row.get("guid"));
+ Assert.assertEquals(row.getString("typeName"), "dsl_test_type");
+ Assert.assertNotNull(row.get("score"));
+
+ int numRows = response.getInt(MetadataServiceClient.COUNT);
+ Assert.assertEquals(numRows, 1);
+ }
+
+ private void createTypes() throws Exception {
+ HierarchicalTypeDefinition<ClassType> dslTestTypeDefinition =
+ TypesUtil.createClassTypeDef("dsl_test_type",
+ ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+
+ HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
+ TypesUtil.createTraitTypeDef("Classification",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
+ HierarchicalTypeDefinition<TraitType> piiTrait =
+ TypesUtil.createTraitTypeDef("PII_TYPE", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> phiTrait =
+ TypesUtil.createTraitTypeDef("PHI", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> pciTrait =
+ TypesUtil.createTraitTypeDef("PCI", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> soxTrait =
+ TypesUtil.createTraitTypeDef("SOX", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> secTrait =
+ TypesUtil.createTraitTypeDef("SEC", ImmutableList.<String>of());
+ HierarchicalTypeDefinition<TraitType> financeTrait =
+ TypesUtil.createTraitTypeDef("Finance", ImmutableList.<String>of());
+
+ TypesDef typesDef = TypeUtils.getTypesDef(
+ ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
+ soxTrait, secTrait, financeTrait),
+ ImmutableList.of(dslTestTypeDefinition));
+ createType(typesDef);
+ }
+
+ private Id createInstance() throws Exception {
+ Referenceable entityInstance = new Referenceable("dsl_test_type",
+ "Classification", "PII_TYPE", "PHI", "PCI", "SOX", "SEC", "Finance");
+ entityInstance.set("name", "foo name");
+ entityInstance.set("description", "bar description");
+
+ Struct traitInstance = (Struct) entityInstance.getTrait("Classification");
+ traitInstance.set("tag", "foundation_etl");
+
+ List<String> traits = entityInstance.getTraits();
+ Assert.assertEquals(traits.size(), 7);
+
+ return createInstance(entityInstance);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java
new file mode 100755
index 0000000..e3f29ae
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.atlas.web.util.Servlets;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+
+/**
+ * Integration tests for Rexster Graph Jersey Resource.
+ */
+@Test
+public class RexsterGraphJerseyResourceIT extends BaseResourceIT {
+
+ @BeforeClass
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Test(enabled = false)
+ public void testGetVertex() throws Exception {
+ // todo: add a vertex before fetching it
+
+ WebResource resource = service
+ .path("api/atlas/graph/vertices")
+ .path("0");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+ String response = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(response);
+ }
+
+ public void testGetVertexWithInvalidId() throws Exception {
+ WebResource resource = service
+ .path("api/atlas/graph/vertices/blah");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+ }
+
+ public void testGetVertexProperties() throws Exception {
+
+ }
+
+ public void testGetVertices() throws Exception {
+
+ }
+
+ public void testGetVertexEdges() throws Exception {
+
+ }
+
+ public void testGetEdge() throws Exception {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
new file mode 100755
index 0000000..bd3cbed
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.resources;
+
+import com.google.common.collect.ImmutableList;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization$;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.atlas.web.util.Servlets;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.Response;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Integration test for types jersey resource.
+ */
+public class TypesJerseyResourceIT extends BaseResourceIT {
+
+ private List<HierarchicalTypeDefinition> typeDefinitions;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ super.setUp();
+
+ typeDefinitions = createHiveTypes();
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ typeDefinitions.clear();
+ }
+
+ @Test
+ public void testSubmit() throws Exception {
+ for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
+ String typesAsJSON = TypesSerialization.toJson(typeDefinition);
+ System.out.println("typesAsJSON = " + typesAsJSON);
+
+ WebResource resource = service
+ .path("api/atlas/types");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ JSONArray typesAdded = response.getJSONArray(MetadataServiceClient.TYPES);
+ Assert.assertEquals(typesAdded.length(), 1);
+ Assert.assertEquals(typesAdded.getJSONObject(0).getString("name"), typeDefinition.typeName);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+ }
+ }
+
+ @Test(dependsOnMethods = "testSubmit")
+ public void testGetDefinition() throws Exception {
+ for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
+ System.out.println("typeName = " + typeDefinition.typeName);
+
+ WebResource resource = service
+ .path("api/atlas/types")
+ .path(typeDefinition.typeName);
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.DEFINITION));
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ String typesJson = response.getString(MetadataServiceClient.DEFINITION);
+ final TypesDef typesDef = TypesSerialization.fromJson(typesJson);
+ List<HierarchicalTypeDefinition<ClassType>> hierarchicalTypeDefinitions = typesDef.classTypesAsJavaList();
+ for(HierarchicalTypeDefinition<ClassType> classType : hierarchicalTypeDefinitions) {
+ for(AttributeDefinition attrDef : classType.attributeDefinitions) {
+ if("name".equals(attrDef.name)) {
+ Assert.assertEquals(attrDef.isIndexable, true);
+ Assert.assertEquals(attrDef.isUnique, true);
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testGetDefinitionForNonexistentType() throws Exception {
+ WebResource resource = service
+ .path("api/atlas/types")
+ .path("blah");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testSubmit")
+ public void testGetTypeNames() throws Exception {
+ WebResource resource = service
+ .path("api/atlas/types");
+
+ ClientResponse clientResponse = resource
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertNotNull(list);
+ }
+
+ @Test
+ public void testGetTraitNames() throws Exception {
+ String[] traitsAdded = addTraits();
+
+ WebResource resource = service
+ .path("api/atlas/types");
+
+ ClientResponse clientResponse = resource
+ .queryParam("type", DataTypes.TypeCategory.TRAIT.name())
+ .accept(Servlets.JSON_MEDIA_TYPE)
+ .type(Servlets.JSON_MEDIA_TYPE)
+ .method(HttpMethod.GET, ClientResponse.class);
+ Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+
+ String responseAsString = clientResponse.getEntity(String.class);
+ Assert.assertNotNull(responseAsString);
+
+ JSONObject response = new JSONObject(responseAsString);
+ Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
+
+ final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
+ Assert.assertNotNull(list);
+ Assert.assertTrue(list.length() >= traitsAdded.length);
+ }
+
+ private String[] addTraits() throws Exception {
+ String[] traitNames = {
+ "class_trait",
+ "secure_trait",
+ "pii_trait",
+ "ssn_trait",
+ "salary_trait",
+ "sox_trait",
+ };
+
+ for (String traitName : traitNames) {
+ HierarchicalTypeDefinition<TraitType> traitTypeDef =
+ TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
+ String json = TypesSerialization$.MODULE$.toJson(traitTypeDef, true);
+ createType(json);
+ }
+
+ return traitNames;
+ }
+
+ private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
+ ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
+
+ HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
+ TypesUtil.createClassTypeDef("database",
+ ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
+ typeDefinitions.add(databaseTypeDefinition);
+
+ HierarchicalTypeDefinition<ClassType> tableTypeDefinition = TypesUtil.createClassTypeDef(
+ "table",
+ ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
+ TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
+ new AttributeDefinition("database",
+ "database", Multiplicity.REQUIRED, false, "database"));
+ typeDefinitions.add(tableTypeDefinition);
+
+ HierarchicalTypeDefinition<TraitType> fetlTypeDefinition = TypesUtil.createTraitTypeDef(
+ "fetl",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
+ typeDefinitions.add(fetlTypeDefinition);
+
+ return typeDefinitions;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
new file mode 100644
index 0000000..ee06cdc
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.web.service;
+
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
+
+public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase{
+ @Test
+ public void testServerConfiguredUsingCredentialProvider() throws Exception {
+ // setup the configuration
+ final PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+ // setup the credential provider
+ setupCredentials();
+
+ SecureEmbeddedServer secureEmbeddedServer = null;
+ try {
+ String appPath = System.getProperty("user.dir") + getWarPath();
+ secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) {
+ @Override
+ protected PropertiesConfiguration getConfiguration() {
+ return configuration;
+ }
+ };
+ secureEmbeddedServer.server.start();
+
+ URL url = new URL("https://localhost:21443/");
+ HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+
+ // test to see whether server is up and root page can be served
+ Assert.assertEquals(connection.getResponseCode(), 200);
+ } finally {
+ secureEmbeddedServer.server.stop();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
new file mode 100755
index 0000000..0f3286f
--- /dev/null
+++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.web.service;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import org.apache.atlas.web.resources.AdminJerseyResourceIT;
+import org.apache.atlas.web.resources.BaseResourceIT;
+import org.apache.atlas.web.resources.EntityJerseyResourceIT;
+import org.apache.atlas.web.resources.MetadataDiscoveryJerseyResourceIT;
+import org.apache.atlas.web.resources.RexsterGraphJerseyResourceIT;
+import org.apache.atlas.web.resources.TypesJerseyResourceIT;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.testng.Assert;
+import org.testng.TestListenerAdapter;
+import org.testng.TestNG;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import javax.ws.rs.core.UriBuilder;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH;
+import static org.apache.atlas.security.SecurityProperties.DEFAULT_KEYSTORE_FILE_LOCATION;
+import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY;
+import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY;
+
+/**
+ * Secure Test class for jersey resources.
+ */
+public class SecureEmbeddedServerITBase {
+
+
+ private SecureEmbeddedServer secureEmbeddedServer;
+ protected String providerUrl;
+ private Path jksPath;
+ protected WebResource service;
+
+ static {
+ //for localhost testing only
+ javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
+ new javax.net.ssl.HostnameVerifier(){
+
+ public boolean verify(String hostname,
+ javax.net.ssl.SSLSession sslSession) {
+ if (hostname.equals("localhost")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ System.setProperty("javax.net.ssl.trustStore", DEFAULT_KEYSTORE_FILE_LOCATION);
+ System.setProperty("javax.net.ssl.trustStorePassword", "keypass");
+ System.setProperty("javax.net.ssl.trustStoreType", "JKS");
+ }
+
+ @BeforeClass
+ public void setupServerURI () throws Exception {
+ BaseResourceIT.baseUrl = "https://localhost:21443";
+ }
+
+ @AfterClass
+ public void resetServerURI() throws Exception {
+ BaseResourceIT.baseUrl = "http://localhost:21000";
+ }
+
+ @BeforeMethod
+ public void setup() throws Exception {
+ jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
+ providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ String baseUrl = "https://localhost:21443/";
+
+ DefaultClientConfig config = new DefaultClientConfig();
+ Client client = Client.create(config);
+ client.resource(UriBuilder.fromUri(baseUrl).build());
+
+ service = client.resource(UriBuilder.fromUri(baseUrl).build());
+ }
+
+ @Test
+ public void testNoConfiguredCredentialProvider() throws Exception {
+
+ try {
+ secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/apache-atlas");
+ WebAppContext webapp = new WebAppContext();
+ webapp.setContextPath("/");
+ webapp.setWar(System.getProperty("user.dir") + getWarPath());
+ secureEmbeddedServer.server.setHandler(webapp);
+
+ secureEmbeddedServer.server.start();
+
+ Assert.fail("Should have thrown an exception");
+ } catch (IOException e) {
+ Assert.assertEquals("No credential provider path configured for storage of certificate store passwords", e.getMessage());
+ } finally {
+ secureEmbeddedServer.server.stop();
+ }
+ }
+
+ @Test
+ public void testMissingEntriesInCredentialProvider() throws Exception {
+ // setup the configuration
+ final PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ try {
+ secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
+ @Override
+ protected PropertiesConfiguration getConfiguration() {
+ return configuration;
+ }
+ };
+ Assert.fail("No entries should generate an exception");
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().startsWith("No credential entry found for"));
+ } finally {
+ secureEmbeddedServer.server.stop();
+ }
+
+ }
+
+ /**
+ * Runs the existing webapp test cases, this time against the initiated secure server instance.
+ * @throws Exception
+ */
+ @Test
+ public void runOtherSuitesAgainstSecureServer() throws Exception {
+ final PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl);
+ // setup the credential provider
+ setupCredentials();
+
+ try {
+ secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/apache-atlas") {
+ @Override
+ protected PropertiesConfiguration getConfiguration() {
+ return configuration;
+ }
+ };
+ WebAppContext webapp = new WebAppContext();
+ webapp.setContextPath("/");
+ webapp.setWar(System.getProperty("user.dir") + getWarPath());
+ secureEmbeddedServer.server.setHandler(webapp);
+
+ secureEmbeddedServer.server.start();
+
+ TestListenerAdapter tla = new TestListenerAdapter();
+ TestNG testng = new TestNG();
+ testng.setTestClasses(new Class[] { AdminJerseyResourceIT.class, EntityJerseyResourceIT.class,
+ MetadataDiscoveryJerseyResourceIT.class, RexsterGraphJerseyResourceIT.class, TypesJerseyResourceIT.class});
+ testng.addListener(tla);
+ testng.run();
+
+ } finally {
+ secureEmbeddedServer.server.stop();
+ }
+
+ }
+
+ protected String getWarPath() {
+ return String.format("/target/atlas-webapp-%s",
+ System.getProperty("project.version", "0.1-incubating-SNAPSHOT"));
+ }
+
+ protected void setupCredentials() throws Exception {
+ Configuration conf = new Configuration(false);
+
+ File file = new File(jksPath.toUri().getPath());
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ // create new aliases
+ try {
+
+ char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ KEYSTORE_PASSWORD_KEY, storepass);
+
+ char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ TRUSTSTORE_PASSWORD_KEY, trustpass);
+
+ char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ SERVER_CERT_PASSWORD_KEY, certpass);
+
+ // write out so that it can be found in checks
+ provider.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+}
[43/58] [abbrv] incubator-atlas git commit: Fix broken python tests
as a result of refactoring to atlas
Posted by ve...@apache.org.
Fix broken python tests as a result of refactoring to atlas
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/b286488c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/b286488c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/b286488c
Branch: refs/heads/master
Commit: b286488cbfa9c31bf57545b3df794a3169d02734
Parents: 4ad5c42 1a7dba9
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Mon Jun 8 21:16:57 2015 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Mon Jun 8 21:16:57 2015 -0700
----------------------------------------------------------------------
addons/hive-bridge/src/bin/import-hive.sh | 4 +--
src/bin/atlas_config.py | 6 ++--
src/bin/atlas_start.py | 8 ++---
src/bin/cputil.py | 4 +--
src/bin/quick_start.py | 6 ++--
src/conf/atlas-env.sh | 49 ++++++++++++++++++++++++++
src/test/python/scripts/TestMetadata.py | 14 ++++----
7 files changed, 70 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
[52/58] [abbrv] incubator-atlas git commit: HDPDGI-69:Vishal: Removed
unnecessary transform of json
Posted by ve...@apache.org.
HDPDGI-69:Vishal: Removed unnecessary transform of json
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/63c86bff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/63c86bff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/63c86bff
Branch: refs/heads/master
Commit: 63c86bff05c420001cc1e6b1b7baacb8e41c937c
Parents: 1efb57b
Author: Vishal Kadam <vi...@gmail.com>
Authored: Tue Jun 16 00:14:14 2015 -0400
Committer: Vishal Kadam <vi...@gmail.com>
Committed: Tue Jun 16 00:14:14 2015 -0400
----------------------------------------------------------------------
dashboard/v2/public/modules/lineage/lineageController.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/63c86bff/dashboard/v2/public/modules/lineage/lineageController.js
----------------------------------------------------------------------
diff --git a/dashboard/v2/public/modules/lineage/lineageController.js b/dashboard/v2/public/modules/lineage/lineageController.js
index 0227552..49ce91e 100755
--- a/dashboard/v2/public/modules/lineage/lineageController.js
+++ b/dashboard/v2/public/modules/lineage/lineageController.js
@@ -30,7 +30,7 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
if (!_.isEmpty(response.results.values.vertices)) {
var allGuids = loadProcess(response.results.values.edges, response.results.values.vertices);
allGuids.then(function(res) {
- guidsList = JSON.parse(res);
+ guidsList = res;
$scope.lineageData = transformData(response.results);
if (callRender) {
render();
@@ -55,7 +55,7 @@ angular.module('dgc.lineage').controller('LineageController', ['$element', '$sco
}
$q.all(urlCalls)
.then(function(results) {
- deferred.resolve(JSON.stringify(results));
+ deferred.resolve(results);
});
return deferred.promise;
}
[22/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/ExpressionTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/ExpressionTest.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/ExpressionTest.scala
deleted file mode 100755
index 3e19fcf..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/ExpressionTest.scala
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import org.apache.hadoop.metadata.query.Expressions._
-import org.apache.hadoop.metadata.repository.BaseTest
-import org.junit.{Before, Test}
-
-class ExpressionTest extends BaseTest {
-
- @Before
- override def setup {
- super.setup
-
- QueryTestsUtils.setupTypes
-
- }
-
- @Test def testClass: Unit = {
- val e = QueryProcessor.validate(_class("DB"))
- println(e)
- }
-
- @Test def testFilter: Unit = {
- val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting"))))
- println(e)
- }
-
- @Test def testSelect: Unit = {
- val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting"))).
- select(id("name"), id("owner")))
- println(e)
- }
-
- @Test def testNegTypeTest: Unit = {
- try {
- val e = QueryProcessor.validate(_class("DB").where(id("name")))
- println(e)
- } catch {
- case e: ExpressionException if e.getMessage.endsWith("expression: DB where name") => ()
- }
- }
-
- @Test def testIsTrait: Unit = {
- val e = QueryProcessor.validate(_class("DB").where(isTrait("JdbcAccess")))
- println(e)
- }
-
- @Test def testIsTraitNegative: Unit = {
- try {
- val e = QueryProcessor.validate(_class("DB").where(isTrait("Jdb")))
- println(e)
- } catch {
- case e: ExpressionException if e.getMessage.endsWith("not a TraitType, expression: is Jdb") => ()
- }
- }
-
- @Test def testhasField: Unit = {
- val e = QueryProcessor.validate(_class("DB").where(hasField("name")))
- println(e)
- }
-
- @Test def testHasFieldNegative: Unit = {
- try {
- val e = QueryProcessor.validate(_class("DB").where(hasField("nam")))
- println(e)
- } catch {
- case e: ExpressionException if e.getMessage.endsWith("not a TraitType, expression: is Jdb") => ()
- }
- }
-
- @Test def testFieldReference: Unit = {
- val e = QueryProcessor.validate(_class("DB").field("Table"))
- println(e)
- }
-
- @Test def testNegFieldReference: Unit = {
- try {
- val e = QueryProcessor.validate(_class("DB").where(_class("LoadProcess").hasField("name")))
- println(e)
- } catch {
- case e: ExpressionException
- if e.getMessage.endsWith("srcType of field doesn't match input type, expression: LoadProcess has name") => ()
- }
- }
-
- @Test def testFieldReferenceRedundant: Unit = {
- val e = QueryProcessor.validate(_class("DB").where(_class("DB").hasField("name")))
- println(e)
- }
-
- @Test def testBackReference: Unit = {
- val e = QueryProcessor.validate(
- _class("DB").as("db1").field("Table").where(id("db1").field("name").`=`(string("Reporting"))))
- println(e)
- }
-
- @Test def testArith: Unit = {
- val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting"))).
- select(id("name"), id("createTime") + int(1)))
- println(e)
- }
-
- @Test def testComparisonLogical: Unit = {
- val e = QueryProcessor.validate(_class("DB").where(id("name").`=`(string("Reporting")).
- and(id("createTime") + int(1) > int(0))))
- println(e)
- }
-
- @Test def testJoinAndSelect1: Unit = {
- val e = QueryProcessor.validate(
- _class("DB").as("db1").field("Table").as("tab").where((id("db1").field("createTime") + int(1) > int(0))
- .and(id("db1").field("name").`=`(string("Reporting")))).select(id("db1").field("name").as("dbName"),
- id("tab").field("name").as("tabName"))
- )
- println(e)
- }
-
- @Test def testJoinAndSelect2: Unit = {
- val e = QueryProcessor.validate(
- _class("DB").as("db1").field("Table").as("tab").where((id("db1").field("createTime") + int(1) > int(0))
- .or(id("db1").field("name").`=`(string("Reporting"))))
- .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName"))
- )
- println(e)
- }
-
- @Test def testJoinAndSelect3: Unit = {
- val e = QueryProcessor.validate(
- _class("DB").as("db1").field("Table").as("tab").where((id("db1").field("createTime") + int(1) > int(0))
- .and(id("db1").field("name").`=`(string("Reporting")))
- .or(id("db1").hasField("owner")))
- .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName"))
- )
- println(e)
- }
-
- @Test def testJoinAndSelect4: Unit = {
- val e = QueryProcessor.validate(
- _class("DB") as "db1" join "Table" as "tab" where (
- id("db1").field("createTime") + int(1) > int(0) and
- (id("db1") `.` "name" `=` string("Reporting")) or
- (id("db1") hasField "owner")
- ) select(
- id("db1") `.` "name" as "dbName", id("tab") `.` "name" as "tabName"
- )
- )
- println(e)
- }
-
- @Test def testLineageAll: Unit = {
- val e = QueryProcessor.validate(_class("Table").loop(id("LoadProcess").field("outputTable")))
- println(e)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest.scala
deleted file mode 100755
index 0880c69..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest.scala
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import com.thinkaurelius.titan.core.TitanGraph
-import org.apache.hadoop.metadata.query.Expressions._
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import org.junit.runner.RunWith
-import org.scalatest._
-import Matchers._
-import org.scalatest.junit.JUnitRunner
-
-@RunWith(classOf[JUnitRunner])
-class GremlinTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
-
- var g: TitanGraph = null
-
- override def beforeAll() {
- TypeSystem.getInstance().reset()
- QueryTestsUtils.setupTypes
- g = QueryTestsUtils.setupTestGraph
- }
-
- override def afterAll() {
- g.shutdown()
- }
-
- test("testClass") {
- val r = QueryProcessor.evaluate(_class("DB"), g)
- validateJson(r, "{\n \"query\":\"DB\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.hadoop.metadata.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"dataTypeName\":\"in
t\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
- }
-
- test("testName") {
- val r = QueryProcessor.evaluate(_class("DB").field("name"), g)
- validateJson(r, "{\n \"query\":\"DB.name\",\n \"dataType\":\"string\",\n \"rows\":[\n \"Sales\",\n \"Reporting\"\n ]\n}")
- }
-
- test("testFilter") {
- var r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))), g)
- validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.hadoop.metadata.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\
",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
- }
-
- test("testFilter2") {
- var r = QueryProcessor.evaluate(_class("DB").where(id("DB").field("name").`=`(string("Reporting"))), g)
- validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\")\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.hadoop.metadata.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\
",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
- }
-
-
- test("testSelect") {
- val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))).
- select(id("name"), id("owner")), g)
- validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _col_0, _src1.owner as _col_1\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct1\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_col_0\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"_col_1\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct1\",\n
\"_col_1\":\"Jane BI\",\n \"_col_0\":\"Reporting\"\n }\n ]\n}")
- }
-
- test("testIsTrait") {
- val r = QueryProcessor.evaluate(_class("Table").where(isTrait("Dimension")), g)
- validateJson(r, "{\n \"query\":\"Table where Table is Dimension\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.hadoop.metadata.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dat
aTypeName\":\"StorageDesc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"
sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n }\n ]\n}")
- }
-
- test("testhasField") {
- val r = QueryProcessor.evaluate(_class("DB").where(hasField("name")), g)
- validateJson(r, "{\n \"query\":\"DB where DB has name\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.hadoop.metadata.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"createTime\",\n \"d
ataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"John ETL\",\n \"name\":\"Sales\",\n \"createTime\":1000\n },\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
- }
-
- test("testFieldReference") {
- val r = QueryProcessor.evaluate(_class("DB").field("Table"), g)
- validateJson(r, "{\n \"query\":\"DB Table\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.hadoop.metadata.typesystem.types.ClassType\",\n \"typeName\":\"Table\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"db\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"sd\",\n \"dataTypeName\":\"StorageD
esc\",\n \"multiplicity\":{\n \"lower\":1,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"2048\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"512\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"2304\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n
\"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"product_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"3840\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"time_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"5376\",\n \"$typeName$\":\"StorageDesc\",\n \"ver
sion\":0\n },\n \"db\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"customer_dim\",\n \"$traits$\":{\n \"Dimension\":{\n \"$typeName$\":\"Dimension\"\n }\n }\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"8960\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"7424\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"Table\",\n \"$id$\":{\n \"id\":\"12800\",\n \"$typeName$\":\"Table\",\n \"version\":0\n },\n \"sd\":{\n \"id\":\"11264\",\n \"$typeName$\":\"StorageDesc\",\n \"version\":0\n },\n \"db\":{\n \"id\
":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"name\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
- }
-
- test("testBackReference") {
- val r = QueryProcessor.evaluate(
- _class("DB").as("db").field("Table").where(id("db").field("name").`=`(string("Reporting"))), g)
- validateJson(r, null)
- }
-
- test("testArith") {
- val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))).
- select(id("name"), id("createTime") + int(1)), g)
- validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _col_0, (_src1.createTime + 1) as _col_1\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct3\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_col_0\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"_col_1\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStru
ct3\",\n \"_col_1\":1501,\n \"_col_0\":\"Reporting\"\n }\n ]\n}")
- }
-
- test("testComparisonLogical") {
- val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting")).
- and(id("createTime") > int(0))), g)
- validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") and (createTime > 0)\",\n \"dataType\":{\n \"superTypes\":[\n \n ],\n \"hierarchicalMetaTypeName\":\"org.apache.hadoop.metadata.typesystem.types.ClassType\",\n \"typeName\":\"DB\",\n \"attributeDefinitions\":[\n {\n \"name\":\"name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"owner\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \
"name\":\"createTime\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"DB\",\n \"$id$\":{\n \"id\":\"7168\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"owner\":\"Jane BI\",\n \"name\":\"Reporting\",\n \"createTime\":1500\n }\n ]\n}")
- }
-
- test("testJoinAndSelect1") {
- val r = QueryProcessor.evaluate(
- _class("DB").as("db1").where(id("name").`=`(string("Sales"))).field("Table").as("tab").
- where((isTrait("Dimension"))).
- select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g
- )
- validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct5\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n
]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
- }
-
- test("testJoinAndSelect2") {
- val r = QueryProcessor.evaluate(
- _class("DB").as("db1").where((id("db1").field("createTime") > int(0))
- .or(id("name").`=`(string("Reporting")))).field("Table").as("tab")
- .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g
- )
- validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\"
:\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
- }
-
- test("testJoinAndSelect3") {
- val r = QueryProcessor.evaluate(
- _class("DB").as("db1").where((id("db1").field("createTime") > int(0))
- .and(id("db1").field("name").`=`(string("Reporting")))
- .or(id("db1").hasField("owner"))).field("Table").as("tab")
- .select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g
- )
- validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or DB as db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\
n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
- }
-
- test("testJoinAndSelect4") {
- val r = QueryProcessor.evaluate(
- _class("DB").as("db1").where(id("name").`=`(string("Sales"))).field("Table").as("tab").
- where((isTrait("Dimension"))).
- select(id("db1").as("dbO"), id("tab").field("name").as("tabName")), g
- )
- validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1 as dbO, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbO\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$
typeName$\":\"\",\n \"dbO\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"id\":\"256\",\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
- }
-
- test("testNegativeInvalidType") {
- val p = new QueryParser
- val e = p("from blah").right.get
- an [ExpressionException] should be thrownBy QueryProcessor.evaluate(e, g)
- }
-
- test("Bug37860") {
- val p = new QueryParser
- val e = p("Table as t where name = 'sales_fact' db where name = 'Sales' and owner = 'John ETL' select t").right.get
- val r = QueryProcessor.evaluate(e, g)
- validateJson(r)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest2.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest2.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest2.scala
deleted file mode 100755
index 4eb5b1a..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/GremlinTest2.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import com.thinkaurelius.titan.core.TitanGraph
-import org.apache.hadoop.metadata.query.Expressions._
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import org.junit.runner.RunWith
-import org.scalatest._
-import Matchers._
-import org.scalatest.junit.JUnitRunner
-
-@RunWith(classOf[JUnitRunner])
-class GremlinTest2 extends FunSuite with BeforeAndAfterAll with BaseGremlinTest {
-
- var g: TitanGraph = null
-
- override def beforeAll() {
- TypeSystem.getInstance().reset()
- QueryTestsUtils.setupTypes
- g = QueryTestsUtils.setupTestGraph
- }
-
- override def afterAll() {
- g.shutdown()
- }
-
- test("testTraitSelect") {
- val r = QueryProcessor.evaluate(_class("Table").as("t").join("Dimension").as("dim").select(id("t"), id("dim")), g)
- validateJson(r, "{\n \"query\":\"Table as t.Dimension as dim select t as _col_0, dim as _col_1\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_col_0\",\n \"dataTypeName\":\"Table\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"_col_1\",\n \"dataTypeName\":\"Dimension\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"\",\n \"_col_1\":{\n \"$typeName$\":\"Dimension\"\n },\n \"_col_0\"
:{\n \"id\":\"3328\",\n \"$typeName$\":\"Table\",\n \"version\":0\n }\n },\n {\n \"$typeName$\":\"\",\n \"_col_1\":{\n \"$typeName$\":\"Dimension\"\n },\n \"_col_0\":{\n \"id\":\"4864\",\n \"$typeName$\":\"Table\",\n \"version\":0\n }\n },\n {\n \"$typeName$\":\"\",\n \"_col_1\":{\n \"$typeName$\":\"Dimension\"\n },\n \"_col_0\":{\n \"id\":\"6656\",\n \"$typeName$\":\"Table\",\n \"version\":0\n }\n }\n ]\n}")
- }
-
- test("testTrait") {
- val r = QueryProcessor.evaluate(_trait("Dimension"), g)
- validateJson(r)
- }
-
- test("testTraitInstance") {
- val r = QueryProcessor.evaluate(_trait("Dimension").traitInstance(), g)
- validateJson(r)
- }
-
- test("testInstanceAddedToFilter") {
- val r = QueryProcessor.evaluate(_trait("Dimension").hasField("typeName"), g)
- validateJson(r)
- }
-
- test("testInstanceFilter") {
- val r = QueryProcessor.evaluate(_trait("Dimension").traitInstance().hasField("name"), g)
- validateJson(r)
- }
-
- test("testLineageWithPath") {
- val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")).path(), g)
- validateJson(r)
- }
-
- test("testLineageAllSelectWithPath") {
- val r = QueryProcessor.evaluate(_class("Table").as("src").loop(id("LoadProcess").field("outputTable")).as("dest").
- select(id("src").field("name").as("srcTable"), id("dest").field("name").as("destTable")).path(), g)
- validateJson(r)
- }
-
- test("testLineageAllSelectWithPathFromParser") {
- val p = new QueryParser
- val e = p("Table as src loop (LoadProcess outputTable) as dest " +
- "select src.name as srcTable, dest.name as destTable withPath").right.get
- //Table as src loop (LoadProcess where LoadProcess.outputTable) as dest select src.name as srcTable, dest.name as destTable withPath
- val r = QueryProcessor.evaluate(e, g)
- validateJson(r)
- }
-
- test("testLineageAllSelectWithPathFromParser2") {
- val p = new QueryParser
-
- val e = p("Table as src loop (`LoadProcess->outputTable` inputTables) as dest " +
- "select src.name as srcTable, dest.name as destTable withPath").right.get
- val r = QueryProcessor.evaluate(e, g)
- validateJson(r)
- }
-
- test("testHighLevelLineage") {
- val r = HiveLineageQuery("Table", "sales_fact_monthly_mv",
- "LoadProcess",
- "inputTables",
- "outputTable",
- None, Some(List("name")), true, GraphPersistenceStrategy1, g).evaluate()
- validateJson(r)
- }
-
- test("testHighLevelLineageReturnGraph") {
- val r = HiveLineageQuery("Table", "sales_fact_monthly_mv",
- "LoadProcess",
- "inputTables",
- "outputTable",
- None, Some(List("name")), true, GraphPersistenceStrategy1, g).graph
-
- println(r.toInstanceJson)
- //validateJson(r)
- }
-
- test("testHighLevelWhereUsed") {
- val r = HiveWhereUsedQuery("Table", "sales_fact",
- "LoadProcess",
- "inputTables",
- "outputTable",
- None, Some(List("name")), true, GraphPersistenceStrategy1, g).evaluate()
- validateJson(r)
- }
-
- test("testHighLevelWhereUsedReturnGraph") {
- val r = HiveWhereUsedQuery("Table", "sales_fact",
- "LoadProcess",
- "inputTables",
- "outputTable",
- None, Some(List("name")), true, GraphPersistenceStrategy1, g).graph
- println(r.toInstanceJson)
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/HiveTitanSample.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/HiveTitanSample.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/HiveTitanSample.scala
deleted file mode 100755
index a7d2751..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/HiveTitanSample.scala
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import java.io.File
-import java.util.UUID
-import java.util.concurrent.atomic.AtomicInteger
-import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
-
-import com.thinkaurelius.titan.core.TitanGraph
-import com.typesafe.config.ConfigFactory
-import org.apache.commons.io.FileUtils
-
-import scala.collection.mutable.ArrayBuffer
-
-object HiveTitanSample {
-
- private var nextVertexId: AtomicInteger = new AtomicInteger(0)
- private var nextEdgeId: AtomicInteger = new AtomicInteger(1000)
-
- trait Vertex {
- val _id: String
-
- def id = _id
- val version = 0
- val guid = s"""${UUID.randomUUID()}""".stripMargin
-
- def addEdge(to: Vertex, label: String, edges: ArrayBuffer[String]): Unit = {
- edges +=
- s"""{"_id" : "${nextEdgeId.incrementAndGet()}", "_type" : "edge", "_inV" : "${to.id}", "_outV" : "$id", "_label" : "$label"}"""
- }
-
- def toGSon(vertices: ArrayBuffer[String],
- edges: ArrayBuffer[String]): Unit = {
-
- val sb = new StringBuilder
- sb.append( s"""{"typeName" : "${this.getClass.getSimpleName}", "_type" : "vertex"""")
-
- this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f =>
- f.setAccessible(true)
- var fV = f.get(this)
- fV = fV match {
- case _: String => s""""$fV""""
- case _ => fV
- }
-
- fV match {
- case x: Vertex => addEdge(x, s"${this.getClass.getSimpleName}.${f.getName}", edges)
- case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex],
- s"${this.getClass.getSimpleName}.${f.getName}", edges))
- case _ => sb.append( s""", "${f.getName}" : $fV""")
- sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $fV""")
- }
- }
-
- this.getClass.getDeclaredFields filter (_.getName == "traits") foreach { f =>
- f.setAccessible(true)
- var traits = f.get(this).asInstanceOf[Option[List[Trait]]]
-
- if (traits.isDefined) {
- val fV = traits.get.map(_.getClass.getSimpleName).mkString(",")
- sb.append( s""", "traitNames" : "$fV"""")
- }
- }
-
- sb.append("}")
- vertices += sb.toString()
- }
- }
-
- trait Trait extends Vertex
-
- trait Struct extends Vertex
-
- trait Instance extends Vertex {
- val traits: Option[List[Trait]]
-
- override def toGSon(vertices: ArrayBuffer[String],
- edges: ArrayBuffer[String]): Unit = {
- super.toGSon(vertices, edges)
-
- if (traits.isDefined) {
- traits.get foreach { t =>
- t.toGSon(vertices, edges)
- addEdge(t, s"${this.getClass.getSimpleName}.${t.getClass.getSimpleName}", edges)
- }
- }
- }
-
- }
-
- case class JdbcAccess(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
-
- case class PII(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
-
- case class Dimension(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
-
- case class Metric(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
-
- case class ETL(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait
-
-
- case class DB(name: String, owner: String, createTime: Int, traits: Option[List[Trait]] = None,
- _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
-
- case class StorageDescriptor(inputFormat: String, outputFormat: String,
- _id: String = "" + nextVertexId.incrementAndGet()) extends Struct
-
- case class Column(name: String, dataType: String, sd: StorageDescriptor,
- traits: Option[List[Trait]] = None,
- _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
-
- case class Table(name: String, db: DB, sd: StorageDescriptor,
- traits: Option[List[Trait]] = None,
- _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
-
- case class TableDef(name: String, db: DB, inputFormat: String, outputFormat: String,
- columns: List[(String, String, Option[List[Trait]])],
- traits: Option[List[Trait]] = None) {
- val sd = StorageDescriptor(inputFormat, outputFormat)
- val colDefs = columns map { c =>
- Column(c._1, c._2, sd, c._3)
- }
- val tablDef = Table(name, db, sd, traits)
-
- def toGSon(vertices: ArrayBuffer[String],
- edges: ArrayBuffer[String]): Unit = {
- sd.toGSon(vertices, edges)
- colDefs foreach {
- _.toGSon(vertices, edges)
- }
- tablDef.toGSon(vertices, edges)
- }
- }
-
- case class LoadProcess(name: String, inputTables: List[Vertex],
- outputTable: Vertex,
- traits: Option[List[Trait]] = None,
- _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
-
- case class View(name: String, db: DB, inputTables: List[Vertex],
- traits: Option[List[Trait]] = None,
- _id: String = "" + nextVertexId.incrementAndGet()) extends Instance
-
- val salesDB = DB("Sales", "John ETL", 1000)
- val salesFact = TableDef("sales_fact",
- salesDB,
- "TextInputFormat",
- "TextOutputFormat",
- List(
- ("time_id", "int", None),
- ("product_id", "int", None),
- ("customer_id", "int", None),
- ("sales", "double", Some(List(Metric())))
- ))
- val productDim = TableDef("product_dim",
- salesDB,
- "TextInputFormat",
- "TextOutputFormat",
- List(
- ("product_id", "int", None),
- ("product_name", "string", None),
- ("brand_name", "string", None)
- ),
- Some(List(Dimension())))
- val timeDim = TableDef("time_dim",
- salesDB,
- "TextInputFormat",
- "TextOutputFormat",
- List(
- ("time_id", "int", None),
- ("dayOfYear", "int", None),
- ("weekDay", "string", None)
- ),
- Some(List(Dimension())))
- val customerDim = TableDef("customer_dim",
- salesDB,
- "TextInputFormat",
- "TextOutputFormat",
- List(
- ("customer_id", "int", None),
- ("name", "int", None),
- ("address", "string", Some(List(PII())))
- ),
- Some(List(Dimension())))
-
- val reportingDB = DB("Reporting", "Jane BI", 1500)
- val salesFactDaily = TableDef("sales_fact_daily_mv",
- reportingDB,
- "TextInputFormat",
- "TextOutputFormat",
- List(
- ("time_id", "int", None),
- ("product_id", "int", None),
- ("customer_id", "int", None),
- ("sales", "double", Some(List(Metric())))
- ))
- val loadSalesFactDaily = LoadProcess("loadSalesDaily",
- List(salesFact.tablDef, timeDim.tablDef), salesFactDaily.tablDef,
- Some(List(ETL())))
-
-
- val productDimView = View("product_dim_view", reportingDB,
- List(productDim.tablDef),
- Some(List(Dimension(), JdbcAccess())))
-
- val customerDimView = View("customer_dim_view", reportingDB,
- List(customerDim.tablDef),
- Some(List(Dimension(), JdbcAccess())))
-
- val salesFactMonthly = TableDef("sales_fact_monthly_mv",
- reportingDB,
- "TextInputFormat",
- "TextOutputFormat",
- List(
- ("time_id", "int", None),
- ("product_id", "int", None),
- ("customer_id", "int", None),
- ("sales", "double", Some(List(Metric())))
- ))
- val loadSalesFactMonthly = LoadProcess("loadSalesMonthly",
- List(salesFactDaily.tablDef), salesFactMonthly.tablDef,
- Some(List(ETL())))
-
-
- val vertices: ArrayBuffer[String] = new ArrayBuffer[String]()
- val edges: ArrayBuffer[String] = new ArrayBuffer[String]()
-
- salesDB.toGSon(vertices, edges)
- salesFact.toGSon(vertices, edges)
- productDim.toGSon(vertices, edges)
- timeDim.toGSon(vertices, edges)
- customerDim.toGSon(vertices, edges)
-
- reportingDB.toGSon(vertices, edges)
- salesFactDaily.toGSon(vertices, edges)
- loadSalesFactDaily.toGSon(vertices, edges)
- productDimView.toGSon(vertices, edges)
- customerDimView.toGSon(vertices, edges)
- salesFactMonthly.toGSon(vertices, edges)
- loadSalesFactMonthly.toGSon(vertices, edges)
-
- def toGSon(): String = {
- s"""{
- "mode":"NORMAL",
- "vertices": ${vertices.mkString("[\n\t", ",\n\t", "\n]")},
- "edges": ${edges.mkString("[\n\t", ",\n\t", "\n]")}
- }
- """.stripMargin
- }
-
- def writeGson(fileName: String): Unit = {
- FileUtils.writeStringToFile(new File(fileName), toGSon())
- }
-
- val GremlinQueries = List(
- // 1. List all DBs
- """g.V.has("typeName", "DB")""",
-
- // 2. List all DB nmes
- """g.V.has("typeName", "DB").name""",
-
- // 3. List all Tables in Reporting DB
- """g.V.has("typeName", "DB").has("name", "Reporting").inE("Table.db").outV""",
- """g.V.has("typeName", "DB").as("db").inE("Table.db").outV.and(_().back("db").has("name", "Reporting"))""",
-
- // 4. List all Tables in Reporting DB, list as D.name, Tbl.name
- """
- g.V.has("typeName", "DB").has("name", "Reporting").as("db").inE("Table.db").outV.as("tbl").select{it.name}{it.name}
- """.stripMargin,
-
- // 5. List all tables that are Dimensions and have the TextInputFormat
- """
- g.V.as("v").and(_().outE("Table.Dimension"), _().out("Table.sd").has("inputFormat", "TextInputFormat")).name
- """.stripMargin,
-
- // 6. List all tables that are Dimensions or have the TextInputFormat
- """
- g.V.as("v").or(_().outE("Table.Dimension"), _().out("Table.sd").has("inputFormat", "TextInputFormat")).name
- """.stripMargin,
-
- // 7. List tables that have at least 1 PII column
- """
- g.V.has("typeName", "Table").as("tab").out("Table.sd").in("Column.sd").as("column"). \
- out("Column.PII").select.groupBy{it.getColumn("tab")}{it.getColumn("column")}{[ "c" : it.size]}.cap.scatter.filter{it.value.c > 0}. \
- transform{it.key}.name """.stripMargin
-
- // 7.a from Table as tab -> g.V.has("typeName", "Table").as("tab")
- // 7.b sd.Column as column -> out("Table.sd").in("Column.sd").as("column")
- // 7.c is PII -> out("Column.PII")
- // 7.d select tab, column -> select{it}{it}
- // 7.e groupBy tab compute count(column) as c
- // 7.f where c > 0
-
- // 7.a Alias(Type("Table"), "tab")
- // 7b. Field("sd", Alias(Type("Table"), "tab"))
- // Alias(Field("Column", Field("sd", Alias(Type("Table"), "tab"))), "column")
- // 7.c Filter(is("PII"), Alias(Field("Column", Field("sd", Alias(Type("Table"), "tab"))), "column"))
- // 7.d
- )
-}
-
-object TestApp extends App with GraphUtils {
-
- var conf = ConfigFactory.load()
- conf = conf.getConfig("graphRepo")
- val g: TitanGraph = titanGraph(conf)
- val manager: ScriptEngineManager = new ScriptEngineManager
- val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
- val bindings: Bindings = engine.createBindings
- bindings.put("g", g)
-
- val hiveGraphFile = FileUtils.getTempDirectory().getPath + File.separator + System.nanoTime() + ".gson"
- HiveTitanSample.writeGson(hiveGraphFile)
- bindings.put("hiveGraphFile", hiveGraphFile)
-
- try {
- engine.eval("g.loadGraphSON(hiveGraphFile)", bindings)
-
- println(engine.eval("g.V.typeName.toList()", bindings))
-
- HiveTitanSample.GremlinQueries.foreach { q =>
- println(q)
- println("Result: " + engine.eval(q + ".toList()", bindings))
- }
- } finally {
- g.shutdown()
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/LexerTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/LexerTest.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/LexerTest.scala
deleted file mode 100755
index d0adc8e..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/LexerTest.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import org.junit.{Assert, Test}
-
-import scala.util.parsing.input.CharArrayReader
-
-class LexerTest {
-
- def scan(p: QueryParser, str: String): p.lexical.ParseResult[_] = {
- val l = p.lexical
- var s: l.Input = new CharArrayReader(str.toCharArray)
- var r = (l.whitespace.? ~ l.token)(s)
- s = r.next
-
- while (r.successful && !s.atEnd) {
- s = r.next
- if (!s.atEnd) {
- r = (l.whitespace.? ~ l.token)(s)
- }
- }
- r.asInstanceOf[p.lexical.ParseResult[_]]
- }
-
- @Test def testSimple {
- val p = new QueryParser
- val r = scan(p, """DB where db1.name""")
- Assert.assertTrue(r.successful)
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/LineageQueryTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/LineageQueryTest.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/LineageQueryTest.scala
deleted file mode 100755
index 15a644f..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/LineageQueryTest.scala
+++ /dev/null
@@ -1,515 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import com.thinkaurelius.titan.core.TitanGraph
-import org.apache.hadoop.metadata.query.Expressions._
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import org.junit.runner.RunWith
-import org.scalatest.junit.JUnitRunner
-import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite}
-
-@RunWith(classOf[JUnitRunner])
-class LineageQueryTest extends FunSuite with BeforeAndAfterAll {
-
- var g: TitanGraph = null
-
- override def beforeAll() {
- TypeSystem.getInstance().reset()
- QueryTestsUtils.setupTypes
- g = QueryTestsUtils.setupTestGraph
- }
-
- override def afterAll() {
- g.shutdown()
- }
-
- val STRUCT_NAME_REGEX = (TypeUtils.TEMP_STRUCT_NAME_PREFIX + "\\d+").r
- val PREFIX_SPACES_REGEX = ("\\n\\s*").r
-
- def validateJson(r: GremlinQueryResult, expected: String = null): Unit = {
- val rJ = r.toJson
- if (expected != null) {
- var a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
- a = PREFIX_SPACES_REGEX.replaceAllIn(a, "")
- var b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
- b = PREFIX_SPACES_REGEX.replaceAllIn(b, "")
- Assertions.assert(a == b)
- } else {
- println(rJ)
- }
- }
-
- test("testInputTables") {
- val r = QueryProcessor.evaluate(_class("LoadProcess").field("inputTables"), g)
- val x = r.toJson
- validateJson(r, """{
- "query":"LoadProcess inputTables",
- "dataType":{
- "superTypes":[
-
- ],
- "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
- "typeName":"Table",
- "attributeDefinitions":[
- {
- "name":"name",
- "dataTypeName":"string",
- "multiplicity":{
- "lower":0,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- },
- {
- "name":"db",
- "dataTypeName":"DB",
- "multiplicity":{
- "lower":1,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- },
- {
- "name":"sd",
- "dataTypeName":"StorageDesc",
- "multiplicity":{
- "lower":1,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- }
- ]
- },
- "rows":[
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"2048",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"512",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"256",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact"
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"4864",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"3840",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"256",
- "$typeName$":"DB",
- "version":0
- },
- "name":"time_dim",
- "$traits$":{
- "Dimension":{
- "$typeName$":"Dimension"
- }
- }
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"8960",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"7424",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_daily_mv"
- }
- ]
-}""")
- }
-
- test("testLoadProcessOut") {
- val r = QueryProcessor.evaluate(_class("Table").field("LoadProcess").field("outputTable"), g)
- validateJson(r, null)
- }
-
- test("testLineageAll") {
- val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")), g)
- validateJson(r, """{
- "query":"Table as _loop0 loop (LoadProcess outputTable)",
- "dataType":{
- "superTypes":[
-
- ],
- "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
- "typeName":"Table",
- "attributeDefinitions":[
- {
- "name":"name",
- "dataTypeName":"string",
- "multiplicity":{
- "lower":0,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- },
- {
- "name":"db",
- "dataTypeName":"DB",
- "multiplicity":{
- "lower":1,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- },
- {
- "name":"sd",
- "dataTypeName":"StorageDesc",
- "multiplicity":{
- "lower":1,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- }
- ]
- },
- "rows":[
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"8960",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"7424",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_daily_mv"
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"12800",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"11264",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_monthly_mv"
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"8960",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"7424",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_daily_mv"
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"12800",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"11264",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_monthly_mv"
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"12800",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"11264",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_monthly_mv"
- }
- ]
-}""")
- }
-
- test("testLineageAllSelect") {
- val r = QueryProcessor.evaluate(_class("Table").as("src").loop(id("LoadProcess").field("outputTable")).as("dest").
- select(id("src").field("name").as("srcTable"), id("dest").field("name").as("destTable")), g)
- validateJson(r, """{
- "query":"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable",
- "dataType":{
- "typeName":"__tempQueryResultStruct2",
- "attributeDefinitions":[
- {
- "name":"srcTable",
- "dataTypeName":"string",
- "multiplicity":{
- "lower":0,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- },
- {
- "name":"destTable",
- "dataTypeName":"string",
- "multiplicity":{
- "lower":0,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- }
- ]
- },
- "rows":[
- {
- "$typeName$":"__tempQueryResultStruct2",
- "srcTable":"sales_fact",
- "destTable":"sales_fact_daily_mv"
- },
- {
- "$typeName$":"__tempQueryResultStruct2",
- "srcTable":"sales_fact",
- "destTable":"sales_fact_monthly_mv"
- },
- {
- "$typeName$":"__tempQueryResultStruct2",
- "srcTable":"time_dim",
- "destTable":"sales_fact_daily_mv"
- },
- {
- "$typeName$":"__tempQueryResultStruct2",
- "srcTable":"time_dim",
- "destTable":"sales_fact_monthly_mv"
- },
- {
- "$typeName$":"__tempQueryResultStruct2",
- "srcTable":"sales_fact_daily_mv",
- "destTable":"sales_fact_monthly_mv"
- }
- ]
-}""")
- }
-
- test("testLineageFixedDepth") {
- val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable"), int(1)), g)
- validateJson(r, """{
- "query":"Table as _loop0 loop (LoadProcess outputTable) times 1",
- "dataType":{
- "superTypes":[
-
- ],
- "hierarchicalMetaTypeName":"org.apache.hadoop.metadata.typesystem.types.ClassType",
- "typeName":"Table",
- "attributeDefinitions":[
- {
- "name":"name",
- "dataTypeName":"string",
- "multiplicity":{
- "lower":0,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- },
- {
- "name":"db",
- "dataTypeName":"DB",
- "multiplicity":{
- "lower":1,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- },
- {
- "name":"sd",
- "dataTypeName":"StorageDesc",
- "multiplicity":{
- "lower":1,
- "upper":1,
- "isUnique":false
- },
- "isComposite":false,
- "isUnique":false,
- "isIndexable":true,
- "reverseAttributeName":null
- }
- ]
- },
- "rows":[
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"8960",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"7424",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_daily_mv"
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"8960",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"7424",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_daily_mv"
- },
- {
- "$typeName$":"Table",
- "$id$":{
- "id":"12800",
- "$typeName$":"Table",
- "version":0
- },
- "sd":{
- "id":"11264",
- "$typeName$":"StorageDesc",
- "version":0
- },
- "db":{
- "id":"7168",
- "$typeName$":"DB",
- "version":0
- },
- "name":"sales_fact_monthly_mv"
- }
- ]
-}""")
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/ParserTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/ParserTest.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/ParserTest.scala
deleted file mode 100755
index 44568a9..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/ParserTest.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import org.apache.hadoop.metadata.repository.BaseTest
-import org.junit.{Before, Test}
-
-
-class ParserTest extends BaseTest {
-
- @Before
- override def setup {
- super.setup
-
- QueryTestsUtils.setupTypes
-
- }
-
- @Test def testFrom: Unit = {
- val p = new QueryParser
- println(p("from DB").right.get.toString)
- }
-
- @Test def testFrom2: Unit = {
- val p = new QueryParser
- println(p("DB").right.get.toString)
- }
-
- @Test def testJoin1: Unit = {
- val p = new QueryParser
- println(p("DB, Table").right.get.toString)
- }
-
- @Test def testWhere1: Unit = {
- val p = new QueryParser
- println(p("DB as db1 Table where db1.name ").right.get.toString)
- }
-
- @Test def testWhere2: Unit = {
- val p = new QueryParser
- println(p("DB name = \"Reporting\"").right.get.toString)
- }
-
- @Test def testIsTrait: Unit = {
- val p = new QueryParser
- println(p("Table isa Dimension").right.get.toString)
- println(p("Table is Dimension").right.get.toString)
- }
-
- @Test def test4: Unit = {
- val p = new QueryParser
- println(p("DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1").right.get.toString)
- }
-
- @Test def testJoin2: Unit = {
- val p = new QueryParser
- println(p("DB as db1 where (createTime + 1) > 0 and (db1.name = \"Reporting\") or DB has owner Table as tab " +
- " select db1.name as dbName, tab.name as tabName").right.get.toString)
- }
-
- @Test def testLoop: Unit = {
- val p = new QueryParser
- println(p("Table loop (LoadProcess outputTable)").right.get.toString)
- }
-
- @Test def testNegInvalidateType: Unit = {
- val p = new QueryParser
- val x = p("from blah")
- println(p("from blah").left)
- }
-
- @Test def testPath1: Unit = {
- val p = new QueryParser
- println(p("Table loop (LoadProcess outputTable) withPath").right.get.toString)
- }
-
- @Test def testPath2: Unit = {
- val p = new QueryParser
- println(p(
- "Table as src loop (LoadProcess outputTable) as dest " +
- "select src.name as srcTable, dest.name as destTable withPath").right.get.toString
- )
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/scala/org/apache/hadoop/metadata/query/QueryTestsUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/hadoop/metadata/query/QueryTestsUtils.scala b/repository/src/test/scala/org/apache/hadoop/metadata/query/QueryTestsUtils.scala
deleted file mode 100755
index 94d66a9..0000000
--- a/repository/src/test/scala/org/apache/hadoop/metadata/query/QueryTestsUtils.scala
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import java.io.File
-import javax.script.{Bindings, ScriptEngine, ScriptEngineManager}
-
-import com.google.common.collect.ImmutableList
-import com.thinkaurelius.titan.core.{TitanFactory, TitanGraph}
-import com.tinkerpop.blueprints.Vertex
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.commons.configuration.{Configuration, ConfigurationException, MapConfiguration}
-import org.apache.commons.io.FileUtils
-import org.apache.hadoop.metadata.typesystem.types._
-import org.scalatest.{BeforeAndAfterAll, FunSuite, Assertions}
-
-trait GraphUtils {
-
- import scala.collection.JavaConversions._
-
- def getConfiguration(config: Config): Configuration = {
- val keys = config.entrySet().map {
- _.getKey
- }
- val gConfig: java.util.Map[String, String] = new java.util.HashMap[String, String]()
- keys.foreach { k =>
- gConfig.put(k, config.getString(k))
- }
- return new MapConfiguration(gConfig)
- }
-
-
- def titanGraph(conf: Config) = {
- try {
- val g = TitanFactory.open(getConfiguration(conf))
- val mgmt = g.getManagementSystem
- val typname = mgmt.makePropertyKey("typeName").dataType(classOf[String]).make()
- mgmt.buildIndex("byTypeName", classOf[Vertex]).addKey(typname).buildCompositeIndex()
- mgmt.commit()
- g
- } catch {
- case e: ConfigurationException => throw new RuntimeException(e)
- }
- }
-}
-
-object QueryTestsUtils extends GraphUtils {
-
- def setupTypes: Unit = {
- def attrDef(name: String, dT: IDataType[_],
- m: Multiplicity = Multiplicity.OPTIONAL,
- isComposite: Boolean = false,
- reverseAttributeName: String = null) = {
- require(name != null)
- require(dT != null)
- new AttributeDefinition(name, dT.getName, m, isComposite, reverseAttributeName)
- }
-
- def dbClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "DB", null,
- Array(
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE)
- ))
-
- def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDesc", null,
- Array(
- attrDef("inputFormat", DataTypes.STRING_TYPE),
- attrDef("outputFormat", DataTypes.STRING_TYPE)
- ))
-
- def columnClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Column", null,
- Array(
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("dataType", DataTypes.STRING_TYPE),
- new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null)
- ))
-
- def tblClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Table", null,
- Array(
- attrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null)
- ))
-
- def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null,
- Array(
- attrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null),
- new AttributeDefinition("outputTable", "Table", Multiplicity.REQUIRED, false, null)
- ))
-
- def viewClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "View", null,
- Array(
- attrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("inputTables", DataTypes.arrayTypeName("Table"), Multiplicity.COLLECTION, false, null)
- ))
-
- def dimTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Dimension", null,
- Array[AttributeDefinition]())
- def piiTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "PII", null,
- Array[AttributeDefinition]())
- def metricTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "Metric", null,
- Array[AttributeDefinition]())
- def etlTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "ETL", null,
- Array[AttributeDefinition]())
- def jdbcTraitDef = new HierarchicalTypeDefinition[TraitType](classOf[TraitType], "JdbcAccess", null,
- Array[AttributeDefinition]())
-
- TypeSystem.getInstance().defineTypes(ImmutableList.of[StructTypeDefinition],
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]](dimTraitDef, piiTraitDef,
- metricTraitDef, etlTraitDef, jdbcTraitDef),
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]](dbClsDef, storageDescClsDef, columnClsDef, tblClsDef,
- loadProcessClsDef, viewClsDef))
-
- ()
- }
-
- def setupTestGraph: TitanGraph = {
- var conf = ConfigFactory.load()
- conf = conf.getConfig("graphRepo")
- val g = titanGraph(conf)
- val manager: ScriptEngineManager = new ScriptEngineManager
- val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy")
- val bindings: Bindings = engine.createBindings
- bindings.put("g", g)
-
- val hiveGraphFile = FileUtils.getTempDirectory().getPath.toString + File.separator + System.nanoTime() + ".gson"
- HiveTitanSample.writeGson(hiveGraphFile)
- bindings.put("hiveGraphFile", hiveGraphFile)
-
- engine.eval("g.loadGraphSON(hiveGraphFile)", bindings)
- g
- }
-
-}
-
-trait BaseGremlinTest {
- self : FunSuite with BeforeAndAfterAll =>
-
- val STRUCT_NAME_REGEX = (TypeUtils.TEMP_STRUCT_NAME_PREFIX + "\\d+").r
- def validateJson(r: GremlinQueryResult, expected: String = null): Unit = {
- val rJ = r.toJson
- if (expected != null) {
- val a = STRUCT_NAME_REGEX.replaceAllIn(rJ, "")
- val b = STRUCT_NAME_REGEX.replaceAllIn(expected, "")
- Assertions.assert(a == b)
- } else {
- println(rJ)
- }
- }
-
-}
[34/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/AttributeStores.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/AttributeStores.java b/repository/src/main/java/org/apache/atlas/repository/memory/AttributeStores.java
new file mode 100755
index 0000000..59cf67d
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/AttributeStores.java
@@ -0,0 +1,655 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
+import it.unimi.dsi.fastutil.bytes.ByteArrayList;
+import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
+import it.unimi.dsi.fastutil.floats.FloatArrayList;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.shorts.ShortArrayList;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.StructInstance;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IConstructableType;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AttributeStores {
+
+ private static final Object NULL_VAL = new Object();
+
+ static IAttributeStore createStore(AttributeInfo i) throws RepositoryException {
+ switch (i.dataType().getTypeCategory()) {
+ case PRIMITIVE:
+ if (i.dataType() == DataTypes.BOOLEAN_TYPE) {
+ return new BooleanAttributeStore(i);
+ } else if (i.dataType() == DataTypes.BYTE_TYPE) {
+ return new ByteAttributeStore(i);
+ } else if (i.dataType() == DataTypes.SHORT_TYPE) {
+ new ShortAttributeStore(i);
+ } else if (i.dataType() == DataTypes.INT_TYPE) {
+ return new IntAttributeStore(i);
+ } else if (i.dataType() == DataTypes.LONG_TYPE) {
+ return new LongAttributeStore(i);
+ } else if (i.dataType() == DataTypes.FLOAT_TYPE) {
+ return new FloatAttributeStore(i);
+ } else if (i.dataType() == DataTypes.DOUBLE_TYPE) {
+ return new DoubleAttributeStore(i);
+ } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ return new BigIntStore(i);
+ } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ return new BigDecimalStore(i);
+ } else if (i.dataType() == DataTypes.DATE_TYPE) {
+ return new DateStore(i);
+ } else if (i.dataType() == DataTypes.STRING_TYPE) {
+ return new StringStore(i);
+ } else if (i.dataType() == DataTypes.STRING_TYPE) {
+ return new StringStore(i);
+ } else {
+ throw new RepositoryException(
+ String.format("Unknown datatype %s", i.dataType()));
+ }
+ case ENUM:
+ return new IntAttributeStore(i);
+ case ARRAY:
+ return new ImmutableListStore(i);
+ case MAP:
+ return new ImmutableMapStore(i);
+ case STRUCT:
+ return new StructStore(i);
+ case CLASS:
+ return new IdStore(i);
+ default:
+ throw new RepositoryException(
+ String.format("Unknown Category for datatype %s", i.dataType()));
+ }
+ }
+
+ static abstract class AbstractAttributeStore implements IAttributeStore {
+ final BooleanArrayList nullList;
+ final Map<Integer, Map<String, Object>> hiddenVals;
+ AttributeInfo attrInfo;
+
+ AbstractAttributeStore(AttributeInfo attrInfo) {
+ this.attrInfo = attrInfo;
+ this.nullList = new BooleanArrayList();
+ hiddenVals = new HashMap<Integer, Map<String, Object>>();
+ }
+
+ final void setNull(int pos, boolean flag) {
+ nullList.set(pos, flag);
+ }
+
+ final boolean getNull(int pos) {
+ return nullList.get(pos);
+ }
+
+ void storeHiddenVals(int pos, IConstructableType type, StructInstance instance)
+ throws RepositoryException {
+ List<String> attrNames = type.getNames(attrInfo);
+ Map<String, Object> m = hiddenVals.get(pos);
+ if (m == null) {
+ m = new HashMap<String, Object>();
+ hiddenVals.put(pos, m);
+ }
+ for (int i = 2; i < attrNames.size(); i++) {
+ String attrName = attrNames.get(i);
+ int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
+ int colPos = instance.fieldMapping().fieldPos.get(attrName);
+ if (instance.nullFlags[nullPos]) {
+ m.put(attrName, NULL_VAL);
+ } else {
+ //m.put(attrName, instance.bools[colPos]);
+ store(instance, colPos, attrName, m);
+ }
+ }
+ }
+
+ void loadHiddenVals(int pos, IConstructableType type, StructInstance instance)
+ throws RepositoryException {
+ List<String> attrNames = type.getNames(attrInfo);
+ Map<String, Object> m = hiddenVals.get(pos);
+ for (int i = 2; i < attrNames.size(); i++) {
+ String attrName = attrNames.get(i);
+ int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
+ int colPos = instance.fieldMapping().fieldPos.get(attrName);
+ Object val = m == null ? NULL_VAL : m.get(attrName);
+ if (val == NULL_VAL) {
+ instance.nullFlags[nullPos] = true;
+ } else {
+ instance.nullFlags[nullPos] = false;
+ load(instance, colPos, val);
+ }
+ }
+ }
+
+ @Override
+ public void store(int pos, IConstructableType type, StructInstance instance)
+ throws RepositoryException {
+ List<String> attrNames = type.getNames(attrInfo);
+ String attrName = attrNames.get(0);
+ int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
+ int colPos = instance.fieldMapping().fieldPos.get(attrName);
+ nullList.set(pos, instance.nullFlags[nullPos]);
+
+ if (pos == nullList.size()) {
+ nullList.add(instance.nullFlags[nullPos]);
+ } else {
+ nullList.set(pos, instance.nullFlags[nullPos]);
+ }
+ //list.set(pos, instance.bools[colPos]);
+ store(instance, colPos, pos);
+
+ if (attrNames.size() > 1) {
+ storeHiddenVals(pos, type, instance);
+ }
+ }
+
+ @Override
+ public void load(int pos, IConstructableType type, StructInstance instance)
+ throws RepositoryException {
+ List<String> attrNames = type.getNames(attrInfo);
+ String attrName = attrNames.get(0);
+ int nullPos = instance.fieldMapping().fieldNullPos.get(attrName);
+ int colPos = instance.fieldMapping().fieldPos.get(attrName);
+
+ if (nullList.get(pos)) {
+ instance.nullFlags[nullPos] = true;
+ } else {
+ instance.nullFlags[nullPos] = false;
+ load(instance, colPos, pos);
+ }
+
+ if (attrNames.size() > 1) {
+ loadHiddenVals(pos, type, instance);
+ }
+ }
+
+ /*
+ * store the value from colPos in instance into the list.
+ */
+ protected abstract void store(StructInstance instance, int colPos, int pos)
+ throws RepositoryException;
+
+ /*
+ * load the value from pos in list into colPos in instance.
+ */
+ protected abstract void load(StructInstance instance, int colPos, int pos)
+ throws RepositoryException;
+
+ /*
+ * store the value from colPos in map as attrName
+ */
+ protected abstract void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m);
+
+ /*
+ * load the val into colPos in instance.
+ */
+ protected abstract void load(StructInstance instance, int colPos, Object val);
+
+ }
+
+ static abstract class PrimitiveAttributeStore extends AbstractAttributeStore
+ implements IAttributeStore {
+
+
+ public PrimitiveAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ }
+
+ }
+
+ static class BooleanAttributeStore extends PrimitiveAttributeStore {
+
+ final BooleanArrayList list;
+
+ BooleanAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = new BooleanArrayList();
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.bools[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.bools[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.bools[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.bools[colPos] = (Boolean) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ list.size(pos + 1);
+ nullList.size(pos + 1);
+ }
+ }
+
+ static class ByteAttributeStore extends PrimitiveAttributeStore {
+
+ final ByteArrayList list;
+
+ ByteAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = new ByteArrayList();
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.bytes[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.bytes[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.bytes[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.bytes[colPos] = (Byte) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ list.size(pos + 1);
+ nullList.size(pos + 1);
+ }
+ }
+
+ static class ShortAttributeStore extends PrimitiveAttributeStore {
+
+ final ShortArrayList list;
+
+ ShortAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = new ShortArrayList();
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.shorts[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.shorts[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.shorts[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.shorts[colPos] = (Short) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ list.size(pos + 1);
+ nullList.size(pos + 1);
+ }
+ }
+
+ static class IntAttributeStore extends PrimitiveAttributeStore {
+
+ final IntArrayList list;
+
+ IntAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = new IntArrayList();
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.ints[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.ints[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.ints[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.ints[colPos] = (Integer) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ list.size(pos + 1);
+ nullList.size(pos + 1);
+ }
+ }
+
+ static class LongAttributeStore extends PrimitiveAttributeStore {
+
+ final LongArrayList list;
+
+ LongAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = new LongArrayList();
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.longs[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.longs[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.longs[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.longs[colPos] = (Long) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ list.size(pos + 1);
+ nullList.size(pos + 1);
+ }
+ }
+
+ static class FloatAttributeStore extends PrimitiveAttributeStore {
+
+ final FloatArrayList list;
+
+ FloatAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = new FloatArrayList();
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.floats[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.floats[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.floats[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.floats[colPos] = (Float) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ list.size(pos + 1);
+ nullList.size(pos + 1);
+ }
+ }
+
+ static class DoubleAttributeStore extends PrimitiveAttributeStore {
+
+ final DoubleArrayList list;
+
+ DoubleAttributeStore(AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = new DoubleArrayList();
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.doubles[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.doubles[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.doubles[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.doubles[colPos] = (Double) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ list.size(pos + 1);
+ nullList.size(pos + 1);
+ }
+ }
+
+ static abstract class ObjectAttributeStore<T> extends AbstractAttributeStore {
+
+ final ArrayList<T> list;
+
+ ObjectAttributeStore(Class<T> cls, AttributeInfo attrInfo) {
+ super(attrInfo);
+ this.list = Lists.newArrayList((T) null);
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ while (list.size() < pos + 1) {
+ list.add((T) null);
+ }
+ nullList.size(pos + 1);
+ }
+ }
+
+ static class BigIntStore extends ObjectAttributeStore<BigInteger> {
+
+ public BigIntStore(AttributeInfo attrInfo) {
+ super(BigInteger.class, attrInfo);
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.bigIntegers[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.bigIntegers[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.bigIntegers[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.bigIntegers[colPos] = (BigInteger) val;
+ }
+
+ }
+
+ static class BigDecimalStore extends ObjectAttributeStore<BigDecimal> {
+
+ public BigDecimalStore(AttributeInfo attrInfo) {
+ super(BigDecimal.class, attrInfo);
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.bigDecimals[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.bigDecimals[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.bigDecimals[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.bigDecimals[colPos] = (BigDecimal) val;
+ }
+
+ }
+
+ static class DateStore extends ObjectAttributeStore<Date> {
+
+ public DateStore(AttributeInfo attrInfo) {
+ super(Date.class, attrInfo);
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.dates[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.dates[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.dates[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.dates[colPos] = (Date) val;
+ }
+
+ }
+
+ static class StringStore extends ObjectAttributeStore<String> {
+
+ public StringStore(AttributeInfo attrInfo) {
+ super(String.class, attrInfo);
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.strings[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.strings[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.strings[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.strings[colPos] = (String) val;
+ }
+
+ }
+
+ static class IdStore extends ObjectAttributeStore<Id> {
+
+ public IdStore(AttributeInfo attrInfo) {
+ super(Id.class, attrInfo);
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.ids[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.ids[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.ids[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.ids[colPos] = (Id) val;
+ }
+
+ }
+
+ static class ImmutableListStore extends ObjectAttributeStore<ImmutableList> {
+
+ public ImmutableListStore(AttributeInfo attrInfo) {
+ super(ImmutableList.class, attrInfo);
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.arrays[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.arrays[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.arrays[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.arrays[colPos] = (ImmutableList) val;
+ }
+
+ }
+
+ static class ImmutableMapStore extends ObjectAttributeStore<ImmutableMap> {
+
+ public ImmutableMapStore(AttributeInfo attrInfo) {
+ super(ImmutableMap.class, attrInfo);
+ }
+
+ protected void store(StructInstance instance, int colPos, int pos) {
+ list.set(pos, instance.maps[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, int pos) {
+ instance.maps[colPos] = list.get(pos);
+ }
+
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.maps[colPos]);
+ }
+
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.maps[colPos] = (ImmutableMap) val;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/ClassStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/ClassStore.java b/repository/src/main/java/org/apache/atlas/repository/memory/ClassStore.java
new file mode 100755
index 0000000..a4933ff
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/ClassStore.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.types.ClassType;
+
+import java.util.ArrayList;
+
+public class ClassStore extends HierarchicalTypeStore {
+
+ final ArrayList<ImmutableList<String>> traitNamesStore;
+ final ClassType classType;
+
+ public ClassStore(MemRepository repository, ClassType hierarchicalType)
+ throws RepositoryException {
+ super(repository, hierarchicalType);
+ classType = hierarchicalType;
+ traitNamesStore = new ArrayList<ImmutableList<String>>();
+ }
+
+ void store(ReferenceableInstance i) throws RepositoryException {
+ super.store(i);
+ int pos = idPosMap.get(i.getId());
+ traitNamesStore.set(pos, i.getTraits());
+ }
+
+ public void ensureCapacity(int pos) throws RepositoryException {
+ super.ensureCapacity(pos);
+ while (traitNamesStore.size() < pos + 1) {
+ traitNamesStore.add(null);
+ }
+ }
+
+ boolean validate(MemRepository repo, Id id) throws RepositoryException {
+ if (id.isUnassigned()) {
+ throw new RepositoryException(String.format("Invalid Id (unassigned) : %s", id));
+ }
+ Integer pos = idPosMap.get(id);
+ if (pos == null) {
+ throw new RepositoryException(String.format("Invalid Id (unknown) : %s", id));
+ }
+
+ String typeName = typeNameList.get(pos);
+ if (typeName != hierarchicalType.getName()) {
+ throw new RepositoryException(
+ String.format("Invalid Id (incorrect typeName, type is %s) : %s",
+ typeName, id));
+ }
+
+ return true;
+ }
+
+ /*
+ * - assumes id is already validated
+ */
+ ReferenceableInstance createInstance(MemRepository repo, Id id) throws RepositoryException {
+ Integer pos = idPosMap.get(id);
+ String typeName = typeNameList.get(pos);
+ if (typeName != hierarchicalType.getName()) {
+ return repo.getClassStore(typeName).createInstance(repo, id);
+ }
+
+ ImmutableList<String> traitNames = traitNamesStore.get(pos);
+ String[] tNs = traitNames.toArray(new String[]{});
+
+ try {
+ ReferenceableInstance r = (ReferenceableInstance) classType.createInstance(id, tNs);
+ return r;
+ } catch (MetadataException me) {
+ throw new RepositoryException(me);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java
new file mode 100755
index 0000000..c9e245b
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/HierarchicalTypeStore.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.StructInstance;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IConstructableType;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public abstract class HierarchicalTypeStore {
+
+ final MemRepository repository;
+ final IConstructableType hierarchicalType;
+ final ArrayList<String> typeNameList;
+ final ImmutableMap<AttributeInfo, IAttributeStore> attrStores;
+ final ImmutableList<HierarchicalTypeStore> superTypeStores;
+
+
+ /**
+ * Map Id to position in storage lists.
+ */
+ Map<Id, Integer> idPosMap;
+
+ List<Integer> freePositions;
+
+ int nextPos;
+
+ /**
+ * Lock for each Class/Trait.
+ */
+ ReentrantReadWriteLock lock;
+
+ HierarchicalTypeStore(MemRepository repository, HierarchicalType hierarchicalType)
+ throws RepositoryException {
+ this.hierarchicalType = (IConstructableType) hierarchicalType;
+ this.repository = repository;
+ ImmutableMap.Builder<AttributeInfo, IAttributeStore> b
+ = new ImmutableBiMap.Builder<AttributeInfo,
+ IAttributeStore>();
+ typeNameList = Lists.newArrayList((String) null);
+ ImmutableList<AttributeInfo> l = hierarchicalType.immediateAttrs;
+ for (AttributeInfo i : l) {
+ b.put(i, AttributeStores.createStore(i));
+ }
+ attrStores = b.build();
+
+ ImmutableList.Builder<HierarchicalTypeStore> b1
+ = new ImmutableList.Builder<HierarchicalTypeStore>();
+ Set<String> allSuperTypeNames = hierarchicalType.getAllSuperTypeNames();
+ for (String s : allSuperTypeNames) {
+ b1.add(repository.getStore(s));
+ }
+ superTypeStores = b1.build();
+
+ nextPos = 0;
+ idPosMap = new HashMap<Id, Integer>();
+ freePositions = new ArrayList<Integer>();
+
+ lock = new ReentrantReadWriteLock();
+ }
+
+ /**
+ * Assign a storage position to an Id.
+ * - try to assign from freePositions
+ * - ensure storage capacity.
+ * - add entry in idPosMap.
+ * @param id
+ * @return
+ * @throws RepositoryException
+ */
+ int assignPosition(Id id) throws RepositoryException {
+
+ int pos = -1;
+ if (!freePositions.isEmpty()) {
+ pos = freePositions.remove(0);
+ } else {
+ pos = nextPos++;
+ ensureCapacity(pos);
+ }
+
+ idPosMap.put(id, pos);
+
+ for (HierarchicalTypeStore s : superTypeStores) {
+ s.assignPosition(id);
+ }
+
+ return pos;
+ }
+
+ /**
+ * - remove from idPosMap
+ * - add to freePositions.
+ * @throws RepositoryException
+ */
+ void releaseId(Id id) {
+
+ Integer pos = idPosMap.get(id);
+ if (pos != null) {
+ idPosMap.remove(id);
+ freePositions.add(pos);
+
+ for (HierarchicalTypeStore s : superTypeStores) {
+ s.releaseId(id);
+ }
+ }
+ }
+
+ void acquireReadLock() {
+ lock.readLock().lock();
+ }
+
+ void acquireWriteLock() {
+ lock.writeLock().lock();
+ }
+
+ void releaseReadLock() {
+ lock.readLock().unlock();
+ }
+
+ void releaseWriteLock() {
+ lock.writeLock().unlock();
+ }
+
+ protected void storeFields(int pos, StructInstance s) throws RepositoryException {
+ for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
+ IAttributeStore attributeStore = e.getValue();
+ attributeStore.store(pos, hierarchicalType, s);
+ }
+ }
+
+ protected void loadFields(int pos, StructInstance s) throws RepositoryException {
+ for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
+ IAttributeStore attributeStore = e.getValue();
+ attributeStore.load(pos, hierarchicalType, s);
+ }
+ }
+
+ /**
+ * - store the typeName
+ * - store the immediate attributes in the respective IAttributeStore
+ * - call store on each SuperType.
+ * @param i
+ * @throws RepositoryException
+ */
+ void store(ReferenceableInstance i) throws RepositoryException {
+ int pos = idPosMap.get(i.getId());
+ typeNameList.set(pos, i.getTypeName());
+ storeFields(pos, i);
+
+ for (HierarchicalTypeStore s : superTypeStores) {
+ s.store(i);
+ }
+ }
+
+ /**
+ * - copy over the immediate attribute values from the respective IAttributeStore
+ * - call load on each SuperType.
+ * @param i
+ * @throws RepositoryException
+ */
+ void load(ReferenceableInstance i) throws RepositoryException {
+ int pos = idPosMap.get(i.getId());
+ loadFields(pos, i);
+
+ for (HierarchicalTypeStore s : superTypeStores) {
+ s.load(i);
+ }
+ }
+
+ public void ensureCapacity(int pos) throws RepositoryException {
+ while (typeNameList.size() < pos + 1) {
+ typeNameList.add(null);
+ }
+ for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
+ IAttributeStore attributeStore = e.getValue();
+ attributeStore.ensureCapacity(pos);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/IAttributeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/IAttributeStore.java b/repository/src/main/java/org/apache/atlas/repository/memory/IAttributeStore.java
new file mode 100755
index 0000000..cce239b
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/IAttributeStore.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.persistence.StructInstance;
+import org.apache.atlas.typesystem.types.IConstructableType;
+
+public interface IAttributeStore {
+ /**
+ * Store the attribute's value from the 'instance' into this store.
+ * @param pos
+ * @param instance
+ * @throws RepositoryException
+ */
+ void store(int pos, IConstructableType type, StructInstance instance)
+ throws RepositoryException;
+
+ /**
+ * load the Instance with the value from position 'pos' for the attribute.
+ * @param pos
+ * @param instance
+ * @throws RepositoryException
+ */
+ void load(int pos, IConstructableType type, StructInstance instance) throws RepositoryException;
+
+ /**
+ * Ensure store have space for the given pos.
+ * @param pos
+ * @throws RepositoryException
+ */
+ void ensureCapacity(int pos) throws RepositoryException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java b/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java
new file mode 100755
index 0000000..0383377
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/MemRepository.java
@@ -0,0 +1,329 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.DiscoverInstances;
+import org.apache.atlas.repository.IRepository;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.MapIds;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class MemRepository implements IRepository {
+
+/*
+ public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd");
+*/
+
+ final TypeSystem typeSystem;
+ /*
+ * A Store for each Class and Trait.
+ */
+ final Map<String, HierarchicalTypeStore> typeStores;
+ final AtomicInteger ID_SEQ = new AtomicInteger(0);
+
+ public MemRepository(TypeSystem typeSystem) {
+ this.typeSystem = typeSystem;
+ this.typeStores = new HashMap<>();
+ }
+
+/*
+ @Override
+ public DateFormat getDateFormat() {
+ return dateFormat;
+ }
+
+ @Override
+ public DateFormat getTimestampFormat() {
+ return timestampFormat;
+ }
+
+ @Override
+ public boolean allowNullsInCollections() {
+ return false;
+ }
+*/
+
+ @Override
+ public Id newId(String typeName) {
+ return new Id("" + ID_SEQ.incrementAndGet(), 0, typeName);
+ }
+
+ /**
+ * 1. traverse the Object Graph from i and create idToNewIdMap : Map[Id, Id],
+ * also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance]
+ * - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType
+ * - traverse Structs
+ * - traverse Traits.
+ * 1b. Ensure that every newId has an associated Instance.
+ * 2. Traverse oldIdToInstance map create newInstances : List[ITypedReferenceableInstance]
+ * - create a ITypedReferenceableInstance.
+ * replace any old References ( ids or object references) with new Ids.
+ * 3. Traverse over newInstances
+ * - ask ClassStore to assign a position to the Id.
+ * - for Instances with Traits, assign a position for each Trait
+ * - invoke store on the nwInstance.
+ *
+ * Recovery:
+ * - on each newInstance, invoke releaseId and delete on its ClassStore and Traits' Stores.
+ *
+ * @param i
+ * @return
+ * @throws org.apache.atlas.repository.RepositoryException
+ */
+ public ITypedReferenceableInstance create(IReferenceableInstance i) throws RepositoryException {
+
+ DiscoverInstances discoverInstances = new DiscoverInstances(this);
+
+ /*
+ * Step 1: traverse the Object Graph from i and create idToNewIdMap : Map[Id, Id],
+ * also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance]
+ * - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType
+ * - traverse Structs
+ * - traverse Traits.
+ */
+ try {
+ new ObjectGraphWalker(typeSystem, discoverInstances, i).walk();
+ } catch (MetadataException me) {
+ throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
+ }
+
+ /*
+ * Step 1b: Ensure that every newId has an associated Instance.
+ */
+ for (Id oldId : discoverInstances.idToNewIdMap.keySet()) {
+ if (!discoverInstances.idToInstanceMap.containsKey(oldId)) {
+ throw new RepositoryException(String.format("Invalid Object Graph: " +
+ "Encountered an unassignedId %s that is not associated with an Instance",
+ oldId));
+ }
+ }
+
+ /* Step 2: Traverse oldIdToInstance map create newInstances :
+ List[ITypedReferenceableInstance]
+ * - create a ITypedReferenceableInstance.
+ * replace any old References ( ids or object references) with new Ids.
+ */
+ List<ITypedReferenceableInstance> newInstances
+ = new ArrayList<ITypedReferenceableInstance>();
+ ITypedReferenceableInstance retInstance = null;
+ Set<ClassType> classTypes = new TreeSet<ClassType>();
+ Set<TraitType> traitTypes = new TreeSet<TraitType>();
+ for (IReferenceableInstance transientInstance : discoverInstances.idToInstanceMap
+ .values()) {
+ try {
+ ClassType cT = typeSystem
+ .getDataType(ClassType.class, transientInstance.getTypeName());
+ ITypedReferenceableInstance newInstance = cT
+ .convert(transientInstance, Multiplicity.REQUIRED);
+ newInstances.add(newInstance);
+
+ classTypes.add(cT);
+ for (String traitName : newInstance.getTraits()) {
+ TraitType tT = typeSystem.getDataType(TraitType.class, traitName);
+ traitTypes.add(tT);
+ }
+
+ if (newInstance.getId() == i.getId()) {
+ retInstance = newInstance;
+ }
+
+ /*
+ * Now replace old references with new Ids
+ */
+ MapIds mapIds = new MapIds(discoverInstances.idToNewIdMap);
+ new ObjectGraphWalker(typeSystem, mapIds, newInstances).walk();
+
+ } catch (MetadataException me) {
+ throw new RepositoryException(
+ String.format("Failed to create Instance(id = %s",
+ transientInstance.getId()), me);
+ }
+ }
+
+ /*
+ * 3. Acquire Class and Trait Storage locks.
+ * - acquire them in a stable order (super before subclass, classes before traits
+ */
+ for (ClassType cT : classTypes) {
+ HierarchicalTypeStore st = typeStores.get(cT.getName());
+ st.acquireWriteLock();
+ }
+
+ for (TraitType tT : traitTypes) {
+ HierarchicalTypeStore st = typeStores.get(tT.getName());
+ st.acquireWriteLock();
+ }
+
+
+ /*
+ * 4. Traverse over newInstances
+ * - ask ClassStore to assign a position to the Id.
+ * - for Instances with Traits, assign a position for each Trait
+ * - invoke store on the nwInstance.
+ */
+ try {
+ for (ITypedReferenceableInstance instance : newInstances) {
+ HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
+ st.assignPosition(instance.getId());
+ for (String traitName : instance.getTraits()) {
+ HierarchicalTypeStore tt = typeStores.get(traitName);
+ tt.assignPosition(instance.getId());
+ }
+ }
+
+ for (ITypedReferenceableInstance instance : newInstances) {
+ HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
+ st.store((ReferenceableInstance) instance);
+ for (String traitName : instance.getTraits()) {
+ HierarchicalTypeStore tt = typeStores.get(traitName);
+ tt.store((ReferenceableInstance) instance);
+ }
+ }
+ } catch (RepositoryException re) {
+ for (ITypedReferenceableInstance instance : newInstances) {
+ HierarchicalTypeStore st = typeStores.get(instance.getTypeName());
+ st.releaseId(instance.getId());
+ }
+ throw re;
+ } finally {
+ for (ClassType cT : classTypes) {
+ HierarchicalTypeStore st = typeStores.get(cT.getName());
+ st.releaseWriteLock();
+ }
+
+ for (TraitType tT : traitTypes) {
+ HierarchicalTypeStore st = typeStores.get(tT.getName());
+ st.releaseWriteLock();
+ }
+ }
+
+ return retInstance;
+ }
+
+ public ITypedReferenceableInstance update(ITypedReferenceableInstance i)
+ throws RepositoryException {
+ throw new RepositoryException("not implemented");
+ }
+
+ public void delete(ITypedReferenceableInstance i) throws RepositoryException {
+ throw new RepositoryException("not implemented");
+ }
+
+ public ITypedReferenceableInstance get(Id id) throws RepositoryException {
+
+ try {
+ ReplaceIdWithInstance replacer = new ReplaceIdWithInstance(this);
+ ObjectGraphWalker walker = new ObjectGraphWalker(typeSystem, replacer);
+ replacer.setWalker(walker);
+ ITypedReferenceableInstance r = getDuringWalk(id, walker);
+ walker.walk();
+ return r;
+ } catch (MetadataException me) {
+ throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
+ }
+ }
+
+ /*
+ * - Id must be valid; Class must be valid.
+ * - Ask ClassStore to createInstance.
+ * - Ask ClassStore to load instance.
+ * - load instance traits
+ * - add to GraphWalker
+ */
+ ITypedReferenceableInstance getDuringWalk(Id id, ObjectGraphWalker walker)
+ throws RepositoryException {
+ ClassStore cS = getClassStore(id.getTypeName());
+ if (cS == null) {
+ throw new RepositoryException(String.format("Unknown Class %s", id.getTypeName()));
+ }
+ cS.validate(this, id);
+ ReferenceableInstance r = cS.createInstance(this, id);
+ cS.load(r);
+ for (String traitName : r.getTraits()) {
+ HierarchicalTypeStore tt = typeStores.get(traitName);
+ tt.load(r);
+ }
+
+ walker.addRoot(r);
+ return r;
+ }
+
+ HierarchicalTypeStore getStore(String typeName) {
+ return typeStores.get(typeName);
+ }
+
+ ClassStore getClassStore(String typeName) {
+ return (ClassStore) getStore(typeName);
+ }
+
+ public void defineClass(ClassType type) throws RepositoryException {
+ HierarchicalTypeStore s = new ClassStore(this, type);
+ typeStores.put(type.getName(), s);
+ }
+
+ public void defineTrait(TraitType type) throws RepositoryException {
+ HierarchicalTypeStore s = new TraitStore(this, type);
+ typeStores.put(type.getName(), s);
+ }
+
+ public void defineTypes(List<HierarchicalType> types) throws RepositoryException {
+ List<TraitType> tTypes = new ArrayList<TraitType>();
+ List<ClassType> cTypes = new ArrayList<ClassType>();
+
+ for (HierarchicalType h : types) {
+ if (h.getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ tTypes.add((TraitType) h);
+ } else {
+ cTypes.add((ClassType) h);
+ }
+ }
+
+ Collections.sort(tTypes);
+ Collections.sort(cTypes);
+
+ for (TraitType tT : tTypes) {
+ defineTrait(tT);
+ }
+
+ for (ClassType cT : cTypes) {
+ defineClass(cT);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java b/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
new file mode 100755
index 0000000..3fa97e5
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class ReplaceIdWithInstance implements ObjectGraphWalker.NodeProcessor {
+
+ public final Map<Id, ITypedReferenceableInstance> idToInstanceMap;
+ final MemRepository repository;
+ ObjectGraphWalker walker;
+
+ public ReplaceIdWithInstance(MemRepository repository) {
+ this.repository = repository;
+ idToInstanceMap = new HashMap<>();
+ }
+
+ void setWalker(ObjectGraphWalker walker) {
+ this.walker = walker;
+ }
+
+ @Override
+ public void processNode(ObjectGraphWalker.Node nd) throws MetadataException {
+ if (nd.attributeName == null) {
+ // do nothing
+ } else if (!nd.aInfo.isComposite || nd.value == null) {
+ // do nothing
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+
+ if (nd.value != null && nd.value instanceof Id) {
+ Id id = (Id) nd.value;
+ ITypedReferenceableInstance r = getInstance(id);
+ nd.instance.set(nd.attributeName, r);
+ }
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ DataTypes.ArrayType aT = (DataTypes.ArrayType) nd.aInfo.dataType();
+ nd.instance.set(nd.attributeName,
+ convertToInstances((ImmutableCollection) nd.value, nd.aInfo.multiplicity, aT));
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ DataTypes.MapType mT = (DataTypes.MapType) nd.aInfo.dataType();
+ nd.instance.set(nd.attributeName,
+ convertToInstances((ImmutableMap) nd.value, nd.aInfo.multiplicity, mT));
+ }
+ }
+
+ ImmutableCollection<?> convertToInstances(ImmutableCollection<?> val,
+ Multiplicity m, DataTypes.ArrayType arrType)
+ throws MetadataException {
+
+ if (val == null ||
+ arrType.getElemType().getTypeCategory() != DataTypes.TypeCategory.CLASS) {
+ return val;
+ }
+
+ ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder()
+ : ImmutableList.builder();
+ Iterator it = val.iterator();
+ while (it.hasNext()) {
+ Object elem = it.next();
+ if (elem instanceof Id) {
+ Id id = (Id) elem;
+ elem = getInstance(id);
+ }
+
+ b.add(elem);
+
+ }
+ return b.build();
+ }
+
+ ImmutableMap<?, ?> convertToInstances(ImmutableMap val, Multiplicity m,
+ DataTypes.MapType mapType)
+ throws MetadataException {
+
+ if (val == null ||
+ (mapType.getKeyType().getTypeCategory() != DataTypes.TypeCategory.CLASS &&
+ mapType.getValueType().getTypeCategory() != DataTypes.TypeCategory.CLASS)) {
+ return val;
+ }
+ ImmutableMap.Builder b = ImmutableMap.builder();
+ Iterator<Map.Entry> it = val.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry elem = it.next();
+ Object oldKey = elem.getKey();
+ Object oldValue = elem.getValue();
+ Object newKey = oldKey;
+ Object newValue = oldValue;
+
+ if (oldKey instanceof Id) {
+ Id id = (Id) elem;
+ ITypedReferenceableInstance r = getInstance(id);
+ }
+
+ if (oldValue instanceof Id) {
+ Id id = (Id) elem;
+ ITypedReferenceableInstance r = getInstance(id);
+ }
+
+ b.put(newKey, newValue);
+ }
+ return b.build();
+ }
+
+ ITypedReferenceableInstance getInstance(Id id) throws MetadataException {
+
+ ITypedReferenceableInstance r = idToInstanceMap.get(id);
+ if (r == null) {
+ r = repository.get(id);
+ idToInstanceMap.put(id, r);
+ walker.addRoot(r);
+ }
+ return r;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/StructStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/StructStore.java b/repository/src/main/java/org/apache/atlas/repository/memory/StructStore.java
new file mode 100755
index 0000000..fb84e13
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/StructStore.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.persistence.StructInstance;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.StructType;
+
+import java.util.Collection;
+import java.util.Map;
+
+public class StructStore extends AttributeStores.AbstractAttributeStore implements IAttributeStore {
+
+ final StructType structType;
+ final ImmutableMap<AttributeInfo, IAttributeStore> attrStores;
+
+ StructStore(AttributeInfo aInfo) throws RepositoryException {
+ super(aInfo);
+ this.structType = (StructType) aInfo.dataType();
+ ImmutableMap.Builder<AttributeInfo, IAttributeStore> b = new ImmutableBiMap.Builder<>();
+ Collection<AttributeInfo> l = structType.fieldMapping.fields.values();
+ for (AttributeInfo i : l) {
+ b.put(i, AttributeStores.createStore(i));
+ }
+ attrStores = b.build();
+
+ }
+
+ @Override
+ protected void store(StructInstance instance, int colPos, int pos) throws RepositoryException {
+ StructInstance s = instance.structs[colPos];
+ for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
+ IAttributeStore attributeStore = e.getValue();
+ attributeStore.store(pos, structType, s);
+ }
+ }
+
+ @Override
+ protected void load(StructInstance instance, int colPos, int pos) throws RepositoryException {
+ StructInstance s = (StructInstance) structType.createInstance();
+ instance.structs[colPos] = s;
+ for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
+ IAttributeStore attributeStore = e.getValue();
+ attributeStore.load(pos, structType, s);
+ }
+ }
+
+ @Override
+ protected void store(StructInstance instance, int colPos, String attrName,
+ Map<String, Object> m) {
+ m.put(attrName, instance.structs[colPos]);
+ }
+
+ @Override
+ protected void load(StructInstance instance, int colPos, Object val) {
+ instance.structs[colPos] = (StructInstance) val;
+ }
+
+ @Override
+ public void ensureCapacity(int pos) throws RepositoryException {
+ for (Map.Entry<AttributeInfo, IAttributeStore> e : attrStores.entrySet()) {
+ IAttributeStore attributeStore = e.getValue();
+ attributeStore.ensureCapacity(pos);
+ }
+ nullList.size(pos + 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/memory/TraitStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/TraitStore.java b/repository/src/main/java/org/apache/atlas/repository/memory/TraitStore.java
new file mode 100755
index 0000000..cd9c8ee
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/TraitStore.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.memory;
+
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.StructInstance;
+import org.apache.atlas.typesystem.types.TraitType;
+
+import java.util.ArrayList;
+
+public class TraitStore extends HierarchicalTypeStore {
+
+ final ArrayList<String> classNameStore;
+
+ public TraitStore(MemRepository repository, TraitType hierarchicalType)
+ throws RepositoryException {
+ super(repository, hierarchicalType);
+ classNameStore = new ArrayList<>();
+ }
+
+ void store(ReferenceableInstance i) throws RepositoryException {
+ int pos = idPosMap.get(i.getId());
+ StructInstance s = (StructInstance) i.getTrait(hierarchicalType.getName());
+ super.storeFields(pos, s);
+ classNameStore.set(pos, i.getTypeName());
+ }
+
+ void load(ReferenceableInstance i) throws RepositoryException {
+ int pos = idPosMap.get(i.getId());
+ StructInstance s = (StructInstance) i.getTrait(hierarchicalType.getName());
+ super.loadFields(pos, s);
+ }
+
+ public void ensureCapacity(int pos) throws RepositoryException {
+ super.ensureCapacity(pos);
+ while (classNameStore.size() < pos + 1) {
+ classNameStore.add(null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
new file mode 100755
index 0000000..0c46b40
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
@@ -0,0 +1,329 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.GraphTransaction;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+@Singleton
+public class GraphBackedTypeStore implements ITypeStore {
+ public static final String VERTEX_TYPE = "typeSystem";
+ private static final String PROPERTY_PREFIX = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "type.";
+ public static final String SUPERTYPE_EDGE_LABEL = PROPERTY_PREFIX + ".supertype";
+
+ private static Logger LOG = LoggerFactory.getLogger(GraphBackedTypeStore.class);
+
+ private final TitanGraph titanGraph;
+
+ @Inject
+ public GraphBackedTypeStore(GraphProvider<TitanGraph> graphProvider) {
+ titanGraph = graphProvider.get();
+ }
+
+ @Override
+ public void store(TypeSystem typeSystem) throws MetadataException {
+ store(typeSystem, ImmutableList.copyOf(typeSystem.getTypeNames()));
+ }
+
+ @Override
+ public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws MetadataException {
+ ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
+ for (String typeName : typeNames) {
+ if (!coreTypes.contains(typeName)) {
+ IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
+ LOG.debug("Processing {}.{} in type store", dataType.getTypeCategory(), dataType.getName());
+ switch (dataType.getTypeCategory()) {
+ case ENUM:
+ storeInGraph((EnumType)dataType);
+ break;
+
+ case STRUCT:
+ StructType structType = (StructType) dataType;
+ storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(),
+ ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableList.<String>of());
+ break;
+
+ case TRAIT:
+ case CLASS:
+ HierarchicalType type = (HierarchicalType) dataType;
+ storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(),
+ type.immediateAttrs, type.superTypes);
+ break;
+
+ default: //Ignore primitive/collection types as they are covered under references
+ break;
+ }
+ }
+ }
+ }
+
+ private void addProperty(Vertex vertex, String propertyName, Object value) {
+ LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
+ vertex.setProperty(propertyName, value);
+ }
+
+ private void storeInGraph(EnumType dataType) {
+ Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName());
+ List<String> values = new ArrayList<>(dataType.values().size());
+ for (EnumValue enumValue : dataType.values()) {
+ String key = getPropertyKey(dataType.getName(), enumValue.value);
+ addProperty(vertex, key, enumValue.ordinal);
+ values.add(enumValue.value);
+ }
+ addProperty(vertex, getPropertyKey(dataType.getName()), values);
+ }
+
+ private String getPropertyKey(String name) {
+ return PROPERTY_PREFIX + name;
+ }
+
+ private String getPropertyKey(String parent, String child) {
+ return PROPERTY_PREFIX + parent + "." + child;
+ }
+
+ private String getEdgeLabel(String parent, String child) {
+ return PROPERTY_PREFIX + "edge." + parent + "." + child;
+ }
+
+ private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName,
+ ImmutableList<AttributeInfo> attributes, ImmutableList<String> superTypes) throws MetadataException {
+ Vertex vertex = createVertex(category, typeName);
+ List<String> attrNames = new ArrayList<>();
+ if (attributes != null) {
+ for (AttributeInfo attribute : attributes) {
+ String propertyKey = getPropertyKey(typeName, attribute.name);
+ try {
+ addProperty(vertex, propertyKey, attribute.toJson());
+ } catch (JSONException e) {
+ throw new StorageException(typeName, e);
+ }
+ attrNames.add(attribute.name);
+ addReferencesForAttribute(typeSystem, vertex, attribute);
+ }
+ }
+ addProperty(vertex, getPropertyKey(typeName), attrNames);
+
+ //Add edges for hierarchy
+ if (superTypes != null) {
+ for (String superTypeName : superTypes) {
+ HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
+ Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName);
+ addEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
+ }
+ }
+ }
+
+ //Add edges for complex attributes
+ private void addReferencesForAttribute(TypeSystem typeSystem, Vertex vertex, AttributeInfo attribute) throws MetadataException {
+ ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
+ List<IDataType> attrDataTypes = new ArrayList<>();
+ IDataType attrDataType = attribute.dataType();
+ String vertexTypeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
+
+ switch (attrDataType.getTypeCategory()) {
+ case ARRAY:
+ String attrType = TypeUtils.parseAsArrayType(attrDataType.getName());
+ IDataType elementType = typeSystem.getDataType(IDataType.class, attrType);
+ attrDataTypes.add(elementType);
+ break;
+
+ case MAP:
+ String[] attrTypes = TypeUtils.parseAsMapType(attrDataType.getName());
+ IDataType keyType = typeSystem.getDataType(IDataType.class, attrTypes[0]);
+ IDataType valueType = typeSystem.getDataType(IDataType.class, attrTypes[1]);
+ attrDataTypes.add(keyType);
+ attrDataTypes.add(valueType);
+ break;
+
+ case ENUM:
+ case STRUCT:
+ case CLASS:
+ attrDataTypes.add(attrDataType);
+ break;
+
+ case PRIMITIVE: //no vertex for primitive type, hence no edge required
+ break;
+
+ default:
+ throw new IllegalArgumentException("Attribute cannot reference instances of type : " + attrDataType.getTypeCategory());
+ }
+
+ for (IDataType attrType : attrDataTypes) {
+ if (!coreTypes.contains(attrType.getName())) {
+ Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName());
+ String label = getEdgeLabel(vertexTypeName, attribute.name);
+ addEdge(vertex, attrVertex, label);
+ }
+ }
+ }
+
+ private void addEdge(Vertex fromVertex, Vertex toVertex, String label) {
+ LOG.debug("Adding edge from {} to {} with label {}" + toString(fromVertex), toString(toVertex), label);
+ titanGraph.addEdge(null, fromVertex, toVertex, label);
+ }
+
+ @Override
+ @GraphTransaction
+ public TypesDef restore() throws MetadataException {
+ //Get all vertices for type system
+ Iterator vertices =
+ titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
+
+ ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
+ ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
+ ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
+ ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
+
+ while (vertices.hasNext()) {
+ Vertex vertex = (Vertex) vertices.next();
+ DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
+ String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
+ LOG.info("Restoring type {}.{}", typeCategory, typeName);
+ switch (typeCategory) {
+ case ENUM:
+ enums.add(getEnumType(vertex));
+ break;
+
+ case STRUCT:
+ AttributeDefinition[] attributes = getAttributes(vertex, typeName);
+ structs.add(new StructTypeDefinition(typeName, attributes));
+ break;
+
+ case CLASS:
+ ImmutableList<String> superTypes = getSuperTypes(vertex);
+ attributes = getAttributes(vertex, typeName);
+ classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes));
+ break;
+
+ case TRAIT:
+ superTypes = getSuperTypes(vertex);
+ attributes = getAttributes(vertex, typeName);
+ traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes));
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unhandled type category " + typeCategory);
+ }
+ }
+ return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
+ }
+
+ private EnumTypeDefinition getEnumType(Vertex vertex) {
+ String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
+ List<EnumValue> enumValues = new ArrayList<>();
+ List<String> values = vertex.getProperty(getPropertyKey(typeName));
+ for (String value : values) {
+ String valueProperty = getPropertyKey(typeName, value);
+ enumValues.add(new EnumValue(value, vertex.<Integer>getProperty(valueProperty)));
+ }
+ return new EnumTypeDefinition(typeName, enumValues.toArray(new EnumValue[enumValues.size()]));
+ }
+
+ private ImmutableList<String> getSuperTypes(Vertex vertex) {
+ List<String> superTypes = new ArrayList<>();
+ Iterator<Edge> edges = vertex.getEdges(Direction.OUT, SUPERTYPE_EDGE_LABEL).iterator();
+ while (edges.hasNext()) {
+ Edge edge = edges.next();
+ superTypes.add((String) edge.getVertex(Direction.IN).getProperty(Constants.TYPENAME_PROPERTY_KEY));
+ }
+ return ImmutableList.copyOf(superTypes);
+ }
+
+ private AttributeDefinition[] getAttributes(Vertex vertex, String typeName) throws MetadataException {
+ List<AttributeDefinition> attributes = new ArrayList<>();
+ List<String> attrNames = vertex.getProperty(getPropertyKey(typeName));
+ if (attrNames != null) {
+ for (String attrName : attrNames) {
+ try {
+ String propertyKey = getPropertyKey(typeName, attrName);
+ attributes.add(AttributeInfo.fromJson((String) vertex.getProperty(propertyKey)));
+ } catch (JSONException e) {
+ throw new MetadataException(e);
+ }
+ }
+ }
+ return attributes.toArray(new AttributeDefinition[attributes.size()]);
+ }
+
+ private String toString(Vertex vertex) {
+ return PROPERTY_PREFIX + "." + vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
+ }
+
+ /**
+ * Find vertex for the given type category and name, else create new vertex
+ * @param category
+ * @param typeName
+ * @return vertex
+ */
+ private Vertex findVertex(DataTypes.TypeCategory category, String typeName) {
+ LOG.debug("Finding vertex for {}.{}", category, typeName);
+
+ Iterator results = titanGraph.query().has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
+ Vertex vertex = null;
+ if (results != null && results.hasNext()) {
+ //There should be just one vertex with the given typeName
+ vertex = (Vertex) results.next();
+ }
+ return vertex;
+ }
+
+ private Vertex createVertex(DataTypes.TypeCategory category, String typeName) {
+ Vertex vertex = findVertex(category, typeName);
+ if (vertex == null) {
+ LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
+ vertex = titanGraph.addVertex(null);
+ addProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); //Mark as type vertex
+ addProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category);
+ addProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName);
+ }
+ return vertex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
new file mode 100755
index 0000000..39f0160
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.TypeSystem;
+
+public interface ITypeStore {
+ /**
+ * Persist the entire type system - insert or update
+ * @param typeSystem type system to persist
+ * @throws StorageException
+ */
+ void store(TypeSystem typeSystem) throws MetadataException;
+
+ /**
+ * Persist the given type in the type system - insert or update
+ * @param typeSystem type system
+ * @param types types to persist
+ * @throws StorageException
+ */
+ void store(TypeSystem typeSystem, ImmutableList<String> types) throws MetadataException;
+
+ /**
+ * Restore all type definitions
+ * @return List of persisted type definitions
+ * @throws org.apache.atlas.MetadataException
+ */
+ TypesDef restore() throws MetadataException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/atlas/repository/typestore/StorageException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/StorageException.java b/repository/src/main/java/org/apache/atlas/repository/typestore/StorageException.java
new file mode 100755
index 0000000..7b9c45c
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/StorageException.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import org.apache.atlas.MetadataException;
+
+public class StorageException extends MetadataException {
+ public StorageException(String type) {
+ super("Failure in typesystem storage for type " + type);
+ }
+
+ public StorageException(String type, Throwable cause) {
+ super("Failure in typesystem storage for type " + type, cause);
+ }
+
+ public StorageException(Throwable cause) {
+ super("Failure in type system storage", cause);
+ }
+}
[47/58] [abbrv] incubator-atlas git commit: Merge branch 'master'
into dal
Posted by ve...@apache.org.
Merge branch 'master' into dal
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f4579b6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f4579b6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f4579b6b
Branch: refs/heads/master
Commit: f4579b6b50b51399cd76261ab48051d5ebe92d93
Parents: e9dacce 3e1637a
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Thu Jun 11 21:27:11 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Thu Jun 11 21:27:11 2015 +0530
----------------------------------------------------------------------
DISCLAIMER.txt | 6 +-
InstallationSteps.txt | 24 +-
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 35 +-
.../hive/model/HiveDataModelGenerator.java | 42 +--
.../org/apache/atlas/hive/hook/HiveHookIT.java | 10 +-
.../hive/hook/SSLAndKerberosHiveHookIT.java | 10 +-
.../apache/atlas/hive/hook/SSLHiveHookIT.java | 10 +-
.../main/java/org/apache/atlas/AtlasClient.java | 368 +++++++++++++++++++
.../org/apache/atlas/AtlasServiceException.java | 45 +++
.../org/apache/atlas/MetadataServiceClient.java | 368 -------------------
.../apache/atlas/MetadataServiceException.java | 45 ---
.../java/org/apache/atlas/PropertiesUtil.java | 8 +-
.../atlas/security/SecureClientUtils.java | 8 +-
docs/src/site/twiki/Configuration.twiki | 26 +-
docs/src/site/twiki/security.twiki | 26 +-
pom.xml | 6 +-
.../atlas/discovery/DiscoveryException.java | 4 +-
.../atlas/discovery/HiveLineageService.java | 18 +-
.../apache/atlas/discovery/LineageService.java | 12 +-
.../graph/DefaultGraphPersistenceStrategy.java | 7 +-
.../graph/GraphBackedDiscoveryService.java | 4 +-
.../atlas/listener/EntityChangeListener.java | 14 +-
.../atlas/listener/TypesChangeListener.java | 8 +-
.../atlas/repository/DiscoverInstances.java | 4 +-
.../apache/atlas/repository/IndexException.java | 4 +-
.../atlas/repository/MetadataRepository.java | 6 +-
.../atlas/repository/RepositoryException.java | 4 +-
.../graph/GraphBackedMetadataRepository.java | 88 ++---
.../graph/GraphBackedSearchIndexer.java | 6 +-
.../repository/graph/TitanGraphProvider.java | 6 +-
.../atlas/repository/memory/ClassStore.java | 4 +-
.../atlas/repository/memory/MemRepository.java | 8 +-
.../memory/ReplaceIdWithInstance.java | 10 +-
.../typestore/GraphBackedTypeStore.java | 69 ++--
.../atlas/repository/typestore/ITypeStore.java | 10 +-
.../repository/typestore/StorageException.java | 4 +-
.../atlas/services/DefaultMetadataService.java | 79 ++--
.../apache/atlas/services/MetadataService.java | 30 +-
.../org/apache/atlas/query/Expressions.scala | 10 +-
.../org/apache/atlas/query/TypeUtils.scala | 10 +-
.../test/java/org/apache/atlas/TestUtils.java | 4 +-
.../org/apache/atlas/repository/BaseTest.java | 10 +-
.../atlas/repository/memory/ClassTest.java | 4 +-
.../atlas/repository/memory/EnumTest.java | 20 +-
.../repository/memory/InstanceE2ETest.java | 14 +-
.../atlas/repository/memory/StorageTest.java | 10 +-
.../atlas/repository/memory/StructTest.java | 8 +-
.../atlas/repository/memory/TraitTest.java | 6 +-
.../typestore/GraphBackedTypeStoreTest.java | 4 +-
.../org/apache/atlas/tools/dsl/package.scala | 2 +-
.../atlas/tools/thrift/ThriftParser.scala | 6 +-
.../atlas/tools/thrift/ThriftTypesGen.scala | 20 +-
.../org/apache/atlas/tools/dsl/DSLTest.scala | 6 +-
.../java/org/apache/atlas/AtlasException.java | 44 +++
.../org/apache/atlas/MetadataException.java | 45 ---
.../org/apache/atlas/TypeNotFoundException.java | 2 +-
.../org/apache/atlas/typesystem/IInstance.java | 8 +-
.../apache/atlas/typesystem/ITypedInstance.java | 48 +--
.../persistence/DownCastStructInstance.java | 8 +-
.../apache/atlas/typesystem/persistence/Id.java | 104 +++---
.../atlas/typesystem/persistence/MapIds.java | 4 +-
.../persistence/ReferenceableInstance.java | 4 +-
.../typesystem/persistence/StructInstance.java | 154 ++++----
.../typesystem/types/AbstractDataType.java | 6 +-
.../atlas/typesystem/types/AttributeInfo.java | 4 +-
.../atlas/typesystem/types/ClassType.java | 24 +-
.../atlas/typesystem/types/DataTypes.java | 34 +-
.../typesystem/types/DownCastFieldMapping.java | 6 +-
.../apache/atlas/typesystem/types/EnumType.java | 4 +-
.../atlas/typesystem/types/FieldMapping.java | 8 +-
.../typesystem/types/HierarchicalType.java | 40 +-
.../typesystem/types/IConstructableType.java | 4 +-
.../atlas/typesystem/types/IDataType.java | 6 +-
.../typesystem/types/ObjectGraphTraversal.java | 16 +-
.../typesystem/types/ObjectGraphWalker.java | 22 +-
.../atlas/typesystem/types/StructType.java | 14 +-
.../atlas/typesystem/types/TraitType.java | 8 +-
.../atlas/typesystem/types/TypeSystem.java | 112 +++---
.../atlas/typesystem/types/TypeUtils.java | 6 +-
.../typesystem/types/TypedStructHandler.java | 6 +-
.../types/ValueConversionException.java | 4 +-
.../typesystem/json/TypesSerialization.scala | 4 +-
.../typesystem/json/SerializationJavaTest.java | 6 +-
.../apache/atlas/typesystem/types/BaseTest.java | 12 +-
.../atlas/typesystem/types/ClassTest.java | 4 +-
.../apache/atlas/typesystem/types/EnumTest.java | 16 +-
.../atlas/typesystem/types/StructTest.java | 6 +-
.../atlas/typesystem/types/TraitTest.java | 6 +-
.../typesystem/types/TypeInheritanceTest.java | 10 +-
.../typesystem/builders/MultiplicityTest.scala | 4 +-
webapp/src/main/java/org/apache/atlas/Main.java | 17 +-
.../org/apache/atlas/examples/QuickStart.java | 10 +-
.../web/filters/AtlasAuthenticationFilter.java | 104 ++++++
.../apache/atlas/web/filters/AuditFilter.java | 4 +-
.../filters/MetadataAuthenticationFilter.java | 104 ------
.../atlas/web/listeners/GuiceServletConfig.java | 12 +-
.../atlas/web/listeners/LoginProcessor.java | 4 +-
.../atlas/web/resources/EntityResource.java | 58 +--
.../web/resources/HiveLineageResource.java | 14 +-
.../resources/MetadataDiscoveryResource.java | 26 +-
.../web/resources/RexsterGraphResource.java | 18 +-
.../atlas/web/resources/TypesResource.java | 30 +-
.../org/apache/atlas/web/util/Servlets.java | 6 +-
.../atlas/web/resources/BaseResourceIT.java | 10 +-
.../web/resources/EntityJerseyResourceIT.java | 54 +--
.../resources/HiveLineageJerseyResourceIT.java | 42 +--
.../MetadataDiscoveryJerseyResourceIT.java | 20 +-
.../web/resources/TypesJerseyResourceIT.java | 25 +-
108 files changed, 1467 insertions(+), 1474 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f4579b6b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --cc repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index ce31412,0371782..86a10f4
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@@ -352,7 -349,7 +349,7 @@@ public class DefaultMetadataService imp
}
private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition)
- throws MetadataException {
- throws AtlasException {
++ throws AtlasException {
try {
Struct traitInstance = InstanceSerialization.fromJsonStruct(
@@@ -418,7 -415,7 +415,7 @@@
}
private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance)
- throws MetadataException {
- throws AtlasException {
++ throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntityAdded(typedInstance);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f4579b6b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f4579b6b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
[30/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
deleted file mode 100755
index c1da867..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/GraphBackedTypeStore.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.typestore;
-
-import com.google.common.collect.ImmutableList;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metadata.GraphTransaction;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.repository.Constants;
-import org.apache.hadoop.metadata.repository.graph.GraphProvider;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.AttributeInfo;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumType;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalType;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.StructType;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.codehaus.jettison.json.JSONException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-@Singleton
-public class GraphBackedTypeStore implements ITypeStore {
- public static final String VERTEX_TYPE = "typeSystem";
- private static final String PROPERTY_PREFIX = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "type.";
- public static final String SUPERTYPE_EDGE_LABEL = PROPERTY_PREFIX + ".supertype";
-
- private static Logger LOG = LoggerFactory.getLogger(GraphBackedTypeStore.class);
-
- private final TitanGraph titanGraph;
-
- @Inject
- public GraphBackedTypeStore(GraphProvider<TitanGraph> graphProvider) {
- titanGraph = graphProvider.get();
- }
-
- @Override
- public void store(TypeSystem typeSystem) throws MetadataException {
- store(typeSystem, ImmutableList.copyOf(typeSystem.getTypeNames()));
- }
-
- @Override
- public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws MetadataException {
- ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
- for (String typeName : typeNames) {
- if (!coreTypes.contains(typeName)) {
- IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
- LOG.debug("Processing {}.{} in type store", dataType.getTypeCategory(), dataType.getName());
- switch (dataType.getTypeCategory()) {
- case ENUM:
- storeInGraph((EnumType)dataType);
- break;
-
- case STRUCT:
- StructType structType = (StructType) dataType;
- storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(),
- ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableList.<String>of());
- break;
-
- case TRAIT:
- case CLASS:
- HierarchicalType type = (HierarchicalType) dataType;
- storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(),
- type.immediateAttrs, type.superTypes);
- break;
-
- default: //Ignore primitive/collection types as they are covered under references
- break;
- }
- }
- }
- }
-
- private void addProperty(Vertex vertex, String propertyName, Object value) {
- LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
- vertex.setProperty(propertyName, value);
- }
-
- private void storeInGraph(EnumType dataType) {
- Vertex vertex = createVertex(dataType.getTypeCategory(), dataType.getName());
- List<String> values = new ArrayList<>(dataType.values().size());
- for (EnumValue enumValue : dataType.values()) {
- String key = getPropertyKey(dataType.getName(), enumValue.value);
- addProperty(vertex, key, enumValue.ordinal);
- values.add(enumValue.value);
- }
- addProperty(vertex, getPropertyKey(dataType.getName()), values);
- }
-
- private String getPropertyKey(String name) {
- return PROPERTY_PREFIX + name;
- }
-
- private String getPropertyKey(String parent, String child) {
- return PROPERTY_PREFIX + parent + "." + child;
- }
-
- private String getEdgeLabel(String parent, String child) {
- return PROPERTY_PREFIX + "edge." + parent + "." + child;
- }
-
- private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName,
- ImmutableList<AttributeInfo> attributes, ImmutableList<String> superTypes) throws MetadataException {
- Vertex vertex = createVertex(category, typeName);
- List<String> attrNames = new ArrayList<>();
- if (attributes != null) {
- for (AttributeInfo attribute : attributes) {
- String propertyKey = getPropertyKey(typeName, attribute.name);
- try {
- addProperty(vertex, propertyKey, attribute.toJson());
- } catch (JSONException e) {
- throw new StorageException(typeName, e);
- }
- attrNames.add(attribute.name);
- addReferencesForAttribute(typeSystem, vertex, attribute);
- }
- }
- addProperty(vertex, getPropertyKey(typeName), attrNames);
-
- //Add edges for hierarchy
- if (superTypes != null) {
- for (String superTypeName : superTypes) {
- HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
- Vertex superVertex = createVertex(superType.getTypeCategory(), superTypeName);
- addEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
- }
- }
- }
-
- //Add edges for complex attributes
- private void addReferencesForAttribute(TypeSystem typeSystem, Vertex vertex, AttributeInfo attribute) throws MetadataException {
- ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
- List<IDataType> attrDataTypes = new ArrayList<>();
- IDataType attrDataType = attribute.dataType();
- String vertexTypeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
-
- switch (attrDataType.getTypeCategory()) {
- case ARRAY:
- String attrType = TypeUtils.parseAsArrayType(attrDataType.getName());
- IDataType elementType = typeSystem.getDataType(IDataType.class, attrType);
- attrDataTypes.add(elementType);
- break;
-
- case MAP:
- String[] attrTypes = TypeUtils.parseAsMapType(attrDataType.getName());
- IDataType keyType = typeSystem.getDataType(IDataType.class, attrTypes[0]);
- IDataType valueType = typeSystem.getDataType(IDataType.class, attrTypes[1]);
- attrDataTypes.add(keyType);
- attrDataTypes.add(valueType);
- break;
-
- case ENUM:
- case STRUCT:
- case CLASS:
- attrDataTypes.add(attrDataType);
- break;
-
- case PRIMITIVE: //no vertex for primitive type, hence no edge required
- break;
-
- default:
- throw new IllegalArgumentException("Attribute cannot reference instances of type : " + attrDataType.getTypeCategory());
- }
-
- for (IDataType attrType : attrDataTypes) {
- if (!coreTypes.contains(attrType.getName())) {
- Vertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName());
- String label = getEdgeLabel(vertexTypeName, attribute.name);
- addEdge(vertex, attrVertex, label);
- }
- }
- }
-
- private void addEdge(Vertex fromVertex, Vertex toVertex, String label) {
- LOG.debug("Adding edge from {} to {} with label {}" + toString(fromVertex), toString(toVertex), label);
- titanGraph.addEdge(null, fromVertex, toVertex, label);
- }
-
- @Override
- @GraphTransaction
- public TypesDef restore() throws MetadataException {
- //Get all vertices for type system
- Iterator vertices =
- titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
-
- ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
- ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
- ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
- ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
-
- while (vertices.hasNext()) {
- Vertex vertex = (Vertex) vertices.next();
- DataTypes.TypeCategory typeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY);
- String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
- LOG.info("Restoring type {}.{}", typeCategory, typeName);
- switch (typeCategory) {
- case ENUM:
- enums.add(getEnumType(vertex));
- break;
-
- case STRUCT:
- AttributeDefinition[] attributes = getAttributes(vertex, typeName);
- structs.add(new StructTypeDefinition(typeName, attributes));
- break;
-
- case CLASS:
- ImmutableList<String> superTypes = getSuperTypes(vertex);
- attributes = getAttributes(vertex, typeName);
- classTypes.add(new HierarchicalTypeDefinition(ClassType.class, typeName, superTypes, attributes));
- break;
-
- case TRAIT:
- superTypes = getSuperTypes(vertex);
- attributes = getAttributes(vertex, typeName);
- traits.add(new HierarchicalTypeDefinition(TraitType.class, typeName, superTypes, attributes));
- break;
-
- default:
- throw new IllegalArgumentException("Unhandled type category " + typeCategory);
- }
- }
- return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
- }
-
- private EnumTypeDefinition getEnumType(Vertex vertex) {
- String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
- List<EnumValue> enumValues = new ArrayList<>();
- List<String> values = vertex.getProperty(getPropertyKey(typeName));
- for (String value : values) {
- String valueProperty = getPropertyKey(typeName, value);
- enumValues.add(new EnumValue(value, vertex.<Integer>getProperty(valueProperty)));
- }
- return new EnumTypeDefinition(typeName, enumValues.toArray(new EnumValue[enumValues.size()]));
- }
-
- private ImmutableList<String> getSuperTypes(Vertex vertex) {
- List<String> superTypes = new ArrayList<>();
- Iterator<Edge> edges = vertex.getEdges(Direction.OUT, SUPERTYPE_EDGE_LABEL).iterator();
- while (edges.hasNext()) {
- Edge edge = edges.next();
- superTypes.add((String) edge.getVertex(Direction.IN).getProperty(Constants.TYPENAME_PROPERTY_KEY));
- }
- return ImmutableList.copyOf(superTypes);
- }
-
- private AttributeDefinition[] getAttributes(Vertex vertex, String typeName) throws MetadataException {
- List<AttributeDefinition> attributes = new ArrayList<>();
- List<String> attrNames = vertex.getProperty(getPropertyKey(typeName));
- if (attrNames != null) {
- for (String attrName : attrNames) {
- try {
- String propertyKey = getPropertyKey(typeName, attrName);
- attributes.add(AttributeInfo.fromJson((String) vertex.getProperty(propertyKey)));
- } catch (JSONException e) {
- throw new MetadataException(e);
- }
- }
- }
- return attributes.toArray(new AttributeDefinition[attributes.size()]);
- }
-
- private String toString(Vertex vertex) {
- return PROPERTY_PREFIX + "." + vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY);
- }
-
- /**
- * Find vertex for the given type category and name, else create new vertex
- * @param category
- * @param typeName
- * @return vertex
- */
- private Vertex findVertex(DataTypes.TypeCategory category, String typeName) {
- LOG.debug("Finding vertex for {}.{}", category, typeName);
-
- Iterator results = titanGraph.query().has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
- Vertex vertex = null;
- if (results != null && results.hasNext()) {
- //There should be just one vertex with the given typeName
- vertex = (Vertex) results.next();
- }
- return vertex;
- }
-
- private Vertex createVertex(DataTypes.TypeCategory category, String typeName) {
- Vertex vertex = findVertex(category, typeName);
- if (vertex == null) {
- LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
- vertex = titanGraph.addVertex(null);
- addProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); //Mark as type vertex
- addProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category);
- addProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName);
- }
- return vertex;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/ITypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/ITypeStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/ITypeStore.java
deleted file mode 100755
index 930ec1e..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/ITypeStore.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.typestore;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-
-public interface ITypeStore {
- /**
- * Persist the entire type system - insert or update
- * @param typeSystem type system to persist
- * @throws StorageException
- */
- void store(TypeSystem typeSystem) throws MetadataException;
-
- /**
- * Persist the given type in the type system - insert or update
- * @param typeSystem type system
- * @param types types to persist
- * @throws StorageException
- */
- void store(TypeSystem typeSystem, ImmutableList<String> types) throws MetadataException;
-
- /**
- * Restore all type definitions
- * @return List of persisted type definitions
- * @throws org.apache.hadoop.metadata.MetadataException
- */
- TypesDef restore() throws MetadataException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/StorageException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/StorageException.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/StorageException.java
deleted file mode 100755
index bb5b2a3..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/typestore/StorageException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.repository.typestore;
-
-import org.apache.hadoop.metadata.MetadataException;
-
-public class StorageException extends MetadataException {
- public StorageException(String type) {
- super("Failure in typesystem storage for type " + type);
- }
-
- public StorageException(String type, Throwable cause) {
- super("Failure in typesystem storage for type " + type, cause);
- }
-
- public StorageException(Throwable cause) {
- super("Failure in type system storage", cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
deleted file mode 100755
index a5b647f..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/services/DefaultMetadataService.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.services;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.metadata.GraphTransaction;
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.ParamChecker;
-import org.apache.hadoop.metadata.classification.InterfaceAudience;
-import org.apache.hadoop.metadata.discovery.SearchIndexer;
-import org.apache.hadoop.metadata.listener.EntityChangeListener;
-import org.apache.hadoop.metadata.listener.TypesChangeListener;
-import org.apache.hadoop.metadata.repository.MetadataRepository;
-import org.apache.hadoop.metadata.repository.typestore.ITypeStore;
-import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance;
-import org.apache.hadoop.metadata.typesystem.ITypedStruct;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Simple wrapper over TypeSystem and MetadataRepository services with hooks
- * for listening to changes to the repository.
- */
-@Singleton
-public class DefaultMetadataService implements MetadataService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(DefaultMetadataService.class);
-
- private final Set<TypesChangeListener> typesChangeListeners = new LinkedHashSet<>();
- private final Set<EntityChangeListener> entityChangeListeners
- = new LinkedHashSet<>();
-
- private final TypeSystem typeSystem;
- private final MetadataRepository repository;
- private final ITypeStore typeStore;
-
- @Inject
- DefaultMetadataService(MetadataRepository repository,
- SearchIndexer searchIndexer, ITypeStore typeStore) throws MetadataException {
- this.typeStore = typeStore;
- this.typeSystem = TypeSystem.getInstance();
- this.repository = repository;
-
- registerListener(searchIndexer);
- restoreTypeSystem();
- }
-
- private void restoreTypeSystem() {
- LOG.info("Restoring type system from the store");
- try {
- TypesDef typesDef = typeStore.restore();
- typeSystem.defineTypes(typesDef);
-
- // restore types before creating super types
- createSuperTypes();
-
- } catch (MetadataException e) {
- throw new RuntimeException(e);
- }
- LOG.info("Restored type system from the store");
- }
-
- private static final AttributeDefinition NAME_ATTRIBUTE =
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE);
- private static final AttributeDefinition DESCRIPTION_ATTRIBUTE =
- TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE);
-
- @InterfaceAudience.Private
- private void createSuperTypes() throws MetadataException {
- if (typeSystem.isRegistered(MetadataServiceClient.DATA_SET_SUPER_TYPE)) {
- return; // this is already registered
- }
-
- HierarchicalTypeDefinition<ClassType> infraType =
- TypesUtil.createClassTypeDef(MetadataServiceClient.INFRASTRUCTURE_SUPER_TYPE,
- ImmutableList.<String>of(), NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE);
-
- HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
- .createClassTypeDef(MetadataServiceClient.DATA_SET_SUPER_TYPE, ImmutableList.<String>of(),
- NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE);
-
- HierarchicalTypeDefinition<ClassType> processType = TypesUtil
- .createClassTypeDef(MetadataServiceClient.PROCESS_SUPER_TYPE, ImmutableList.<String>of(),
- NAME_ATTRIBUTE, DESCRIPTION_ATTRIBUTE, new AttributeDefinition("inputs",
- DataTypes.arrayTypeName(MetadataServiceClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("outputs",
- DataTypes.arrayTypeName(MetadataServiceClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null));
-
- TypesDef typesDef = TypeUtils
- .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.of(infraType, datasetType, processType));
- createType(TypesSerialization.toJson(typesDef));
- }
-
- /**
- * Creates a new type based on the type system to enable adding
- * entities (instances for types).
- *
- * @param typeDefinition definition as json
- * @return a unique id for this type
- */
- @Override
- @GraphTransaction
- public JSONObject createType(String typeDefinition) throws MetadataException {
- ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty");
-
- TypesDef typesDef;
- try {
- typesDef = TypesSerialization.fromJson(typeDefinition);
- if(typesDef.isEmpty()) {
- throw new MetadataException("Invalid type definition");
- }
- } catch (Exception e) {
- LOG.error("Unable to deserialize json={}", typeDefinition, e);
- throw new IllegalArgumentException("Unable to deserialize json");
- }
-
- try {
- final Map<String, IDataType> typesAdded = typeSystem.defineTypes(typesDef);
-
- try {
- typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
- onTypesAddedToRepo(typesAdded);
- } catch(Throwable t) {
- typeSystem.removeTypes(ImmutableList.copyOf(typesAdded.keySet()));
- throw new MetadataException(t);
- }
-
- return new JSONObject() {{
- put(MetadataServiceClient.TYPES, typesAdded.keySet());
- }};
- } catch (JSONException e) {
- LOG.error("Unable to create response for types={}", typeDefinition, e);
- throw new MetadataException("Unable to create response");
- }
- }
-
- /**
- * Return the definition for the given type.
- *
- * @param typeName name for this type, must be unique
- * @return type definition as JSON
- */
- @Override
- public String getTypeDefinition(String typeName) throws MetadataException {
- final IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
- return TypesSerialization.toJson(typeSystem, dataType.getName());
- }
-
- /**
- * Return the list of types in the repository.
- *
- * @return list of type names in the repository
- */
- @Override
- public List<String> getTypeNamesList() throws MetadataException {
- return typeSystem.getTypeNames();
- }
-
- /**
- * Return the list of trait type names in the type system.
- *
- * @return list of trait type names in the type system
- */
- @Override
- public List<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) throws MetadataException {
- return typeSystem.getTypeNamesByCategory(typeCategory);
- }
-
- /**
- * Creates an entity, instance of the type.
- *
- * @param entityInstanceDefinition definition
- * @return guid
- */
- @Override
- public String createEntity(String entityInstanceDefinition) throws MetadataException {
- ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition cannot be empty");
-
- ITypedReferenceableInstance entityTypedInstance =
- deserializeClassInstance(entityInstanceDefinition);
-
- final String guid = repository.createEntity(entityTypedInstance);
-
- onEntityAddedToRepo(entityTypedInstance);
- return guid;
- }
-
- private ITypedReferenceableInstance deserializeClassInstance(
- String entityInstanceDefinition) throws MetadataException {
-
- final Referenceable entityInstance;
- try {
- entityInstance = InstanceSerialization.fromJsonReferenceable(
- entityInstanceDefinition, true);
- } catch (Exception e) { // exception from deserializer
- LOG.error("Unable to deserialize json={}", entityInstanceDefinition, e);
- throw new IllegalArgumentException("Unable to deserialize json");
- }
- final String entityTypeName = entityInstance.getTypeName();
- ParamChecker.notEmpty(entityTypeName, "Entity type cannot be null");
-
- ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
- return entityType.convert(entityInstance, Multiplicity.REQUIRED);
- }
-
- /**
- * Return the definition for the given guid.
- *
- * @param guid guid
- * @return entity definition as JSON
- */
- @Override
- public String getEntityDefinition(String guid) throws MetadataException {
- ParamChecker.notEmpty(guid, "guid cannot be null");
-
- final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
- return InstanceSerialization.toJson(instance, true);
- }
-
- /**
- * Return the list of entity names for the given type in the repository.
- *
- * @param entityType type
- * @return list of entity names for the given type in the repository
- */
- @Override
- public List<String> getEntityList(String entityType) throws MetadataException {
- validateTypeExists(entityType);
-
- return repository.getEntityList(entityType);
- }
-
- @Override
- public void updateEntity(String guid, String property, String value) throws MetadataException {
- ParamChecker.notEmpty(guid, "guid cannot be null");
- ParamChecker.notEmpty(property, "property cannot be null");
- ParamChecker.notEmpty(value, "property value cannot be null");
-
- repository.updateEntity(guid, property, value);
- }
-
- private void validateTypeExists(String entityType) throws MetadataException {
- ParamChecker.notEmpty(entityType, "entity type cannot be null");
-
- // verify if the type exists
- if (!typeSystem.isRegistered(entityType)) {
- throw new MetadataException("type is not defined for : " + entityType);
- }
- }
-
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- * @throws MetadataException
- */
- @Override
- public List<String> getTraitNames(String guid) throws MetadataException {
- ParamChecker.notEmpty(guid, "entity GUID cannot be null");
- return repository.getTraitNames(guid);
- }
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitInstanceDefinition trait instance json that needs to be added to entity
- * @throws MetadataException
- */
- @Override
- public void addTrait(String guid,
- String traitInstanceDefinition) throws MetadataException {
- ParamChecker.notEmpty(guid, "entity GUID cannot be null");
- ParamChecker.notEmpty(traitInstanceDefinition, "Trait instance cannot be null");
-
- ITypedStruct traitInstance = deserializeTraitInstance(traitInstanceDefinition);
- final String traitName = traitInstance.getTypeName();
-
- // ensure trait type is already registered with the TS
- Preconditions.checkArgument(typeSystem.isRegistered(traitName),
- "trait=%s should be defined in type system before it can be added", traitName);
- // ensure trait is not already defined
- Preconditions.checkArgument(!getTraitNames(guid).contains(traitName),
- "trait=%s is already defined for entity=%s", traitName, guid);
-
- repository.addTrait(guid, traitInstance);
-
- onTraitAddedToEntity(guid, traitName);
- }
-
- private ITypedStruct deserializeTraitInstance(String traitInstanceDefinition)
- throws MetadataException {
-
- try {
- Struct traitInstance = InstanceSerialization.fromJsonStruct(
- traitInstanceDefinition, true);
- final String entityTypeName = traitInstance.getTypeName();
- ParamChecker.notEmpty(entityTypeName, "entity type cannot be null");
-
- TraitType traitType = typeSystem.getDataType(TraitType.class, entityTypeName);
- return traitType.convert(
- traitInstance, Multiplicity.REQUIRED);
- } catch (Exception e) {
- throw new MetadataException("Error deserializing trait instance", e);
- }
- }
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitNameToBeDeleted name of the trait
- * @throws MetadataException
- */
- @Override
- public void deleteTrait(String guid,
- String traitNameToBeDeleted) throws MetadataException {
- ParamChecker.notEmpty(guid, "entity GUID cannot be null");
- ParamChecker.notEmpty(traitNameToBeDeleted, "Trait name cannot be null");
-
- // ensure trait type is already registered with the TS
- Preconditions.checkArgument(typeSystem.isRegistered(traitNameToBeDeleted),
- "trait=%s should be defined in type system before it can be deleted",
- traitNameToBeDeleted);
-
- repository.deleteTrait(guid, traitNameToBeDeleted);
-
- onTraitDeletedFromEntity(guid, traitNameToBeDeleted);
- }
-
- private void onTypesAddedToRepo(Map<String, IDataType> typesAdded) throws MetadataException {
- for (TypesChangeListener listener : typesChangeListeners) {
- for (Map.Entry<String, IDataType> entry : typesAdded.entrySet()) {
- listener.onAdd(entry.getKey(), entry.getValue());
- }
- }
- }
-
- public void registerListener(TypesChangeListener listener) {
- typesChangeListeners.add(listener);
- }
-
- public void unregisterListener(TypesChangeListener listener) {
- typesChangeListeners.remove(listener);
- }
-
- private void onEntityAddedToRepo(ITypedReferenceableInstance typedInstance)
- throws MetadataException {
-
- for (EntityChangeListener listener : entityChangeListeners) {
- listener.onEntityAdded(typedInstance);
- }
- }
-
- private void onTraitAddedToEntity(String typeName,
- String traitName) throws MetadataException {
- for (EntityChangeListener listener : entityChangeListeners) {
- listener.onTraitAdded(typeName, traitName);
- }
- }
-
- private void onTraitDeletedFromEntity(String typeName,
- String traitName) throws MetadataException {
- for (EntityChangeListener listener : entityChangeListeners) {
- listener.onTraitDeleted(typeName, traitName);
- }
- }
-
- public void registerListener(EntityChangeListener listener) {
- entityChangeListeners.add(listener);
- }
-
- public void unregisterListener(EntityChangeListener listener) {
- entityChangeListeners.remove(listener);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java b/repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java
deleted file mode 100755
index 3c43590..0000000
--- a/repository/src/main/java/org/apache/hadoop/metadata/services/MetadataService.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.services;
-
-import org.apache.hadoop.metadata.MetadataException;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.util.List;
-
-/**
- * Metadata service.
- */
-public interface MetadataService {
-
- /**
- * Creates a new type based on the type system to enable adding
- * entities (instances for types).
- *
- * @param typeDefinition definition as json
- * @return a unique id for this type
- */
- JSONObject createType(String typeDefinition) throws MetadataException;
-
- /**
- * Return the definition for the given type.
- *
- * @param typeName name for this type, must be unique
- * @return type definition as JSON
- */
- String getTypeDefinition(String typeName) throws MetadataException;
-
- /**
- * Return the list of types in the type system.
- *
- * @return list of type names in the type system
- */
- List<String> getTypeNamesList() throws MetadataException;
-
- /**
- * Return the list of trait type names in the type system.
- *
- * @return list of trait type names in the type system
- */
- List<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) throws MetadataException;
-
- /**
- * Creates an entity, instance of the type.
- *
- * @param entityDefinition definition
- * @return guid
- */
- String createEntity(String entityDefinition) throws MetadataException;
-
- /**
- * Return the definition for the given guid.
- *
- * @param guid guid
- * @return entity definition as JSON
- */
- String getEntityDefinition(String guid) throws MetadataException;
-
- /**
- * Return the list of entity names for the given type in the repository.
- *
- * @param entityType type
- * @return list of entity names for the given type in the repository
- */
- List<String> getEntityList(String entityType) throws MetadataException;
-
- /**
- * Adds the property to the given entity id(guid).
- *
- * @param guid entity id
- * @param property property name
- * @param value property value
- */
- void updateEntity(String guid, String property, String value) throws MetadataException;
-
- // Trait management functions
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- * @throws MetadataException
- */
- List<String> getTraitNames(String guid) throws MetadataException;
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitInstanceDefinition trait instance that needs to be added to entity
- * @throws MetadataException
- */
- void addTrait(String guid,
- String traitInstanceDefinition) throws MetadataException;
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitNameToBeDeleted name of the trait
- * @throws MetadataException
- */
- void deleteTrait(String guid,
- String traitNameToBeDeleted) throws MetadataException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/ClosureQuery.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/ClosureQuery.scala b/repository/src/main/scala/org/apache/atlas/query/ClosureQuery.scala
new file mode 100755
index 0000000..05dc6a4
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/ClosureQuery.scala
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import java.util
+
+import com.thinkaurelius.titan.core.TitanGraph
+import org.apache.atlas.query.Expressions._
+import org.apache.atlas.typesystem.ITypedStruct
+import org.apache.atlas.typesystem.json.{InstanceSerialization, Serialization}
+import org.apache.atlas.typesystem.persistence.StructInstance
+import org.apache.atlas.typesystem.types.DataTypes.{MapType, PrimitiveType}
+import org.apache.atlas.typesystem.types.{DataTypes, StructType, TypeSystem}
+
+/**
+ * Represents a Query to compute the closure based on a relationship between entities of a particular type.
+ * For e.g. Database Tables are related to each other to capture the '''Lineage''' of data in a Table based
+ * on other Tables.
+ *
+ * A Closure Query is specified by the following information:
+ * - The Type whose instances are in a closure relationship. For e.g. 'Table'
+ * - The Closure relation. This is specified as an ''Attribute path''. For e.g. if we have the following model:
+ * {{{
+ * class Table {
+ * name : String,
+ * ...
+ * }
+ *
+ * class LoadTableProcess {
+ * name : String,
+ * inputTables : List[Table],
+ * outputTable : Table,
+ * ...
+ * }
+ * }}}
+ * ''LoadTable'' instance captures the relationship between the data in an output Table and a set of input Tables.
+ * In order to compute the '''Lineage''' of a Table, the ''Attribute path'' that relates 2 Tables is
+ * '''[(LoadTableProcess,outputTable), inputTables]'''. This list is saying that for any Table I want to connect to other
+ * tables via the LoadProcess.outputTable attribute, and then via the inputTables attribute. So each entry in the
+ * Attribute Path represents an attribute in an object. For reverse relations the Type and attribute must be specified,
+ * as in 'LoadTableProcess,outputTable)', whereas for forward relations the attribute name is sufficient.
+ * - The depth of the traversal. Certain times you are not interested in the complete closure, but to only
+ * discover related instances up to a certain depth. Specify the depth as number of hops, or you can ask for the
+ * complete closure.
+ * - You can ask for certain attributes to be returned. For e.g. you may only want the Table name, owner and
+ * creationDate. By default only the Ids of the related instances is returned.
+ * - For pair of related instances, you optionally ask for the Path of the relation to be returned. This is
+ * returned as a list of ''Id''s.
+ *
+ * Given these 5 things the ClosureQuery can be executed, it returns a GremlinQueryResult of the Closure Query.
+ */
+trait ClosureQuery {
+
+ val SRC_PREFIX = TypeUtils.GraphResultStruct.SRC_PREFIX
+ val DEST_PREFIX = TypeUtils.GraphResultStruct.DEST_PREFIX
+
+ sealed trait PathAttribute {
+
+ def toExpr : Expression = this match {
+ case r : Relation => id(r.attributeName)
+ case rr : ReverseRelation => id(s"${rr.typeName}->${rr.attributeName}")
+ }
+
+ def toFieldName : String = this match {
+ case r : Relation => r.attributeName
+ case rr : ReverseRelation => rr.typeName
+ }
+ }
+ case class ReverseRelation(typeName : String, attributeName : String) extends PathAttribute
+ case class Relation(attributeName : String) extends PathAttribute
+
+ /**
+ * Type on whose instances the closure needs to be computed
+ * @return
+ */
+ def closureType : String
+
+ /**
+ * specify how instances are related.
+ */
+ def closureRelation : List[PathAttribute]
+
+ /**
+ * The maximum hops between related instances. A [[None]] implies there is maximum.
+ * @return
+ */
+ def depth : Option[Int]
+
+ /**
+ * The attributes to return for the instances. These will be prefixed by 'src_' and 'dest_' in the
+ * output rows.
+ * @return
+ */
+ def selectAttributes : Option[List[String]]
+
+ /**
+ * specify if the Path should be returned.
+ * @return
+ */
+ def withPath : Boolean
+
+ def persistenceStrategy: GraphPersistenceStrategies
+ def g: TitanGraph
+
+ def pathExpr : Expressions.Expression = {
+ closureRelation.tail.foldLeft(closureRelation.head.toExpr)((b,a) => b.field(a.toFieldName))
+ }
+
+ def selectExpr(alias : String) : List[Expression] = {
+ selectAttributes.map { _.map { a =>
+ id(alias).field(a).as(s"${alias}_$a")
+ }
+ }.getOrElse(List(id(alias)))
+ }
+
+ /**
+ * hook to allow a filter to be added for the closureType
+ * @param expr
+ * @return
+ */
+ def srcCondition(expr : Expression) : Expression = expr
+
+ def expr : Expressions.Expression = {
+ val e = srcCondition(Expressions._class(closureType)).as(SRC_PREFIX).loop(pathExpr).as(DEST_PREFIX).
+ select((selectExpr(SRC_PREFIX) ++ selectExpr(DEST_PREFIX)):_*)
+ if (withPath) e.path else e
+ }
+
+ def evaluate(): GremlinQueryResult = {
+ var e = expr
+ QueryProcessor.evaluate(e, g, persistenceStrategy)
+ }
+
+ def graph : GraphResult = {
+
+ if (!withPath) {
+ throw new ExpressionException(expr, "Graph requested for non Path Query")
+ }
+
+ import scala.collection.JavaConverters._
+
+ val res = evaluate()
+
+ val graphResType = TypeUtils.GraphResultStruct.createType(res.resultDataType.asInstanceOf[StructType])
+ val vertexPayloadType = {
+ val mT = graphResType.fieldMapping.fields.get(TypeUtils.GraphResultStruct.verticesAttrName).
+ dataType().asInstanceOf[MapType]
+ mT.getValueType.asInstanceOf[StructType]
+ }
+
+ def id(idObj : StructInstance) : String = idObj.getString(TypeSystem.ID_STRUCT_ID_ATTRNAME)
+
+ def vertexStruct(idObj : StructInstance, resRow : ITypedStruct, attrPrefix : String) : StructInstance = {
+ val vP = vertexPayloadType.createInstance()
+ vP.set(TypeUtils.GraphResultStruct.vertexIdAttrName, idObj)
+ vertexPayloadType.fieldMapping.fields.asScala.keys.
+ filter(_ != TypeUtils.GraphResultStruct.vertexIdAttrName).foreach{a =>
+ vP.set(a, resRow.get(s"${attrPrefix}$a"))
+ }
+ vP.asInstanceOf[StructInstance]
+ }
+
+ val instance = graphResType.createInstance()
+ val vertices = new util.HashMap[String, AnyRef]()
+ val edges = new util.HashMap[String,java.util.List[String]]()
+
+ /**
+ * foreach resultRow
+ * for each Path entry
+ * add an entry in the edges Map
+ * add an entry for the Src Vertex to the vertex Map
+ * add an entry for the Dest Vertex to the vertex Map
+ */
+ res.rows.map(_.asInstanceOf[StructInstance]).foreach { r =>
+
+ val path = r.get(TypeUtils.ResultWithPathStruct.pathAttrName).asInstanceOf[java.util.List[_]].asScala
+ val srcVertex = path.head.asInstanceOf[StructInstance]
+
+ var currVertex = srcVertex
+ path.tail.foreach { n =>
+ val nextVertex = n.asInstanceOf[StructInstance]
+ val iList = if (!edges.containsKey(id(currVertex))) {
+ val l = new util.ArrayList[String]()
+ edges.put(id(currVertex), l)
+ l
+ } else {
+ edges.get(id(currVertex))
+ }
+ if ( !iList.contains(id(nextVertex))) {
+ iList.add(id(nextVertex))
+ }
+ currVertex = nextVertex
+ }
+ val vertex = r.get(TypeUtils.ResultWithPathStruct.resultAttrName)
+ vertices.put(id(srcVertex), vertexStruct(srcVertex,
+ r.get(TypeUtils.ResultWithPathStruct.resultAttrName).asInstanceOf[ITypedStruct],
+ s"${SRC_PREFIX}_"))
+ vertices.put(id(currVertex), vertexStruct(currVertex,
+ r.get(TypeUtils.ResultWithPathStruct.resultAttrName).asInstanceOf[ITypedStruct],
+ s"${DEST_PREFIX}_"))
+ }
+
+ instance.set(TypeUtils.GraphResultStruct.verticesAttrName, vertices)
+ instance.set(TypeUtils.GraphResultStruct.edgesAttrName, edges)
+ GraphResult(res.query, instance)
+ }
+}
+
+/**
+ * Closure for a single instance. Instance is specified by an ''attributeToSelectInstance'' and the value
+ * for the attribute.
+ *
+ * @tparam T
+ */
+trait SingleInstanceClosureQuery[T] extends ClosureQuery {
+
+ def attributeToSelectInstance : String
+
+ def attributeTyp : PrimitiveType[T]
+ def instanceValue : T
+
+ override def srcCondition(expr : Expression) : Expression = {
+ expr.where(
+ Expressions.id(attributeToSelectInstance).`=`(Expressions.literal(attributeTyp, instanceValue))
+ )
+ }
+}
+
+/**
+ * A ClosureQuery to compute '''Lineage''' for Hive tables. Assumes the Lineage relation is captured in a ''CTAS''
+ * type, and the table relations are captured as attributes from a CTAS instance to Table instances.
+ *
+ * @param tableTypeName The name of the Table Type.
+ * @param ctasTypeName The name of the Create Table As Select(CTAS) Type.
+ * @param ctasInputTableAttribute The attribute in CTAS Type that associates it to the ''Input'' tables.
+ * @param ctasOutputTableAttribute The attribute in CTAS Type that associates it to the ''Output'' tables.
+ * @param depth depth as needed by the closure Query.
+ * @param selectAttributes as needed by the closure Query.
+ * @param withPath as needed by the closure Query.
+ * @param persistenceStrategy as needed to evaluate the Closure Query.
+ * @param g as needed to evaluate the Closure Query.
+ */
+case class HiveLineageQuery(tableTypeName : String,
+ tableName : String,
+ ctasTypeName : String,
+ ctasInputTableAttribute : String,
+ ctasOutputTableAttribute : String,
+ depth : Option[Int],
+ selectAttributes : Option[List[String]],
+ withPath : Boolean,
+ persistenceStrategy: GraphPersistenceStrategies,
+ g: TitanGraph
+ ) extends SingleInstanceClosureQuery[String] {
+
+ val closureType : String = tableTypeName
+
+ val attributeToSelectInstance = "name"
+ val attributeTyp = DataTypes.STRING_TYPE
+
+ val instanceValue = tableName
+
+ lazy val closureRelation = List(
+ ReverseRelation(ctasTypeName, ctasOutputTableAttribute),
+ Relation(ctasInputTableAttribute)
+ )
+}
+
+/**
+ * A ClosureQuery to compute where a table is used based on the '''Lineage''' for Hive tables.
+ * Assumes the Lineage relation is captured in a ''CTAS''
+ * type, and the table relations are captured as attributes from a CTAS instance to Table instances.
+ *
+ * @param tableTypeName The name of the Table Type.
+ * @param ctasTypeName The name of the Create Table As Select(CTAS) Type.
+ * @param ctasInputTableAttribute The attribute in CTAS Type that associates it to the ''Input'' tables.
+ * @param ctasOutputTableAttribute The attribute in CTAS Type that associates it to the ''Output'' tables.
+ * @param depth depth as needed by the closure Query.
+ * @param selectAttributes as needed by the closure Query.
+ * @param withPath as needed by the closure Query.
+ * @param persistenceStrategy as needed to evaluate the Closure Query.
+ * @param g as needed to evaluate the Closure Query.
+ */
+case class HiveWhereUsedQuery(tableTypeName : String,
+ tableName : String,
+ ctasTypeName : String,
+ ctasInputTableAttribute : String,
+ ctasOutputTableAttribute : String,
+ depth : Option[Int],
+ selectAttributes : Option[List[String]],
+ withPath : Boolean,
+ persistenceStrategy: GraphPersistenceStrategies,
+ g: TitanGraph
+ ) extends SingleInstanceClosureQuery[String] {
+
+ val closureType : String = tableTypeName
+
+ val attributeToSelectInstance = "name"
+ val attributeTyp = DataTypes.STRING_TYPE
+
+ val instanceValue = tableName
+
+ lazy val closureRelation = List(
+ ReverseRelation(ctasTypeName, ctasInputTableAttribute),
+ Relation(ctasOutputTableAttribute)
+ )
+}
+
+case class GraphResult(query: String, result : ITypedStruct) {
+
+ def toTypedJson = Serialization.toJson(result)
+
+ def toInstanceJson = InstanceSerialization.toJson(result)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/atlas/query/Expressions.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/Expressions.scala b/repository/src/main/scala/org/apache/atlas/query/Expressions.scala
new file mode 100755
index 0000000..ed57a91
--- /dev/null
+++ b/repository/src/main/scala/org/apache/atlas/query/Expressions.scala
@@ -0,0 +1,739 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.query
+
+import org.apache.atlas.MetadataException
+import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
+import org.apache.atlas.typesystem.types._
+
+object Expressions {
+
+ import TypeUtils._
+
+ class ExpressionException(val e: Expression, message: String, cause: Throwable, enableSuppression: Boolean,
+ writableStackTrace: Boolean)
+ extends MetadataException(message, cause, enableSuppression, writableStackTrace) {
+
+ def this(e: Expression, message: String) {
+ this(e, message, null, false, false)
+ }
+
+ def this(e: Expression, message: String, cause: Throwable) {
+ this(e, message, cause, false, false)
+ }
+
+ def this(e: Expression, cause: Throwable) {
+ this(e, null, cause, false, false)
+ }
+
+ override def getMessage: String = {
+ val eString = e.toString
+ s"${super.getMessage}, expression:${if (eString contains "\n") "\n" else " "}$e"
+ }
+
+ }
+
+ class UnresolvedException(expr: Expression, function: String) extends
+ ExpressionException(expr, s"Unresolved $function")
+
+ def attachExpression[A](e: Expression, msg: String = "")(f: => A): A = {
+ try f catch {
+ case eex: ExpressionException => throw eex
+ case ex: Exception => throw new ExpressionException(e, msg, ex)
+ }
+ }
+
+ trait Expression {
+ self: Product =>
+
+ def children: Seq[Expression]
+
+ /**
+ * Returns `true` if the schema for this expression and all its children have been resolved.
+ * The default logic is that an Expression is resolve if all its children are resolved.
+ */
+ lazy val resolved: Boolean = childrenResolved
+
+ /**
+ * Returns the output [[IDataType[_]] of this expression. Expressions that are unresolved will
+ * throw if this method is invoked.
+ */
+ def dataType: IDataType[_]
+
+ /**
+ * Returns true if all the children have been resolved.
+ */
+ def childrenResolved = !children.exists(!_.resolved)
+
+
+ /**
+ * the aliases that are present in this Expression Tree
+ */
+ def namedExpressions: Map[String, Expression] = Map()
+
+ def fastEquals(other: Expression): Boolean = {
+ this.eq(other) || this == other
+ }
+
+ def makeCopy(newArgs: Array[AnyRef]): this.type = attachExpression(this, "makeCopy") {
+ try {
+ val defaultCtor = getClass.getConstructors.find(_.getParameterTypes.size != 0).head
+ defaultCtor.newInstance(newArgs: _*).asInstanceOf[this.type]
+ } catch {
+ case e: java.lang.IllegalArgumentException =>
+ throw new ExpressionException(
+ this, s"Failed to copy node. Reason: ${e.getMessage}.")
+ }
+ }
+
+ def transformChildrenDown(rule: PartialFunction[Expression, Expression]): this.type = {
+ var changed = false
+ val newArgs = productIterator.map {
+ case arg: Expression if children contains arg =>
+ val newChild = arg.asInstanceOf[Expression].transformDown(rule)
+ if (!(newChild fastEquals arg)) {
+ changed = true
+ newChild
+ } else {
+ arg
+ }
+ case Some(arg: Expression) if children contains arg =>
+ val newChild = arg.asInstanceOf[Expression].transformDown(rule)
+ if (!(newChild fastEquals arg)) {
+ changed = true
+ Some(newChild)
+ } else {
+ Some(arg)
+ }
+ case m: Map[_, _] => m
+ case args: Traversable[_] => args.map {
+ case arg: Expression if children contains arg =>
+ val newChild = arg.asInstanceOf[Expression].transformDown(rule)
+ if (!(newChild fastEquals arg)) {
+ changed = true
+ newChild
+ } else {
+ arg
+ }
+ case other => other
+ }
+ case nonChild: AnyRef => nonChild
+ case null => null
+ }.toArray
+ if (changed) makeCopy(newArgs) else this
+ }
+
+ def transformDown(rule: PartialFunction[Expression, Expression]): Expression = {
+ val afterRule = rule.applyOrElse(this, identity[Expression])
+ // Check if unchanged and then possibly return old copy to avoid gc churn.
+ if (this fastEquals afterRule) {
+ transformChildrenDown(rule)
+ } else {
+ afterRule.transformChildrenDown(rule)
+ }
+ }
+
+ def traverseChildren(traverseFunc: (Expression, PartialFunction[Expression, Unit]) => Unit)
+ (rule: PartialFunction[Expression, Unit]): Unit = {
+ productIterator.foreach {
+ case arg: Expression if children contains arg =>
+ traverseFunc(arg.asInstanceOf[Expression], rule)
+ case Some(arg: Expression) if children contains arg =>
+ traverseFunc(arg.asInstanceOf[Expression], rule)
+ case m: Map[_, _] => m
+ case args: Traversable[_] => args.map {
+ case arg: Expression if children contains arg =>
+ traverseFunc(arg.asInstanceOf[Expression], rule)
+ case other => other
+ }
+ case nonChild: AnyRef => nonChild
+ case null => null
+ }
+ }
+
+ def traverseChildrenDown = traverseChildren(_traverseDown) _
+
+ private def _traverseDown(e: Expression, rule: PartialFunction[Expression, Unit]): Unit = {
+ if (rule.isDefinedAt(e)) {
+ rule.apply(e)
+ }
+ e.traverseChildrenDown(rule)
+ }
+
+ def traverseDown(rule: PartialFunction[Expression, Unit]): Unit = {
+ _traverseDown(this, rule)
+ }
+
+ def traverseChildrenUp = traverseChildren(_traverseUp) _
+
+ private def _traverseUp(e: Expression, rule: PartialFunction[Expression, Unit]): Unit = {
+ e.traverseChildrenUp(rule)
+ if (rule.isDefinedAt(e)) {
+ rule.apply(e)
+ }
+ }
+
+ def traverseUp(rule: PartialFunction[Expression, Unit]): Unit = {
+ _traverseUp(this, rule)
+ }
+
+ def transformUp(rule: PartialFunction[Expression, Expression]): Expression = {
+ val afterRuleOnChildren = transformChildrenUp(rule);
+ if (this fastEquals afterRuleOnChildren) {
+ rule.applyOrElse(this, identity[Expression])
+ } else {
+ rule.applyOrElse(afterRuleOnChildren, identity[Expression])
+ }
+ }
+
+ def transformChildrenUp(rule: PartialFunction[Expression, Expression]): this.type = {
+ var changed = false
+ val newArgs = productIterator.map {
+ case arg: Expression if children contains arg =>
+ val newChild = arg.asInstanceOf[Expression].transformUp(rule)
+ if (!(newChild fastEquals arg)) {
+ changed = true
+ newChild
+ } else {
+ arg
+ }
+ case Some(arg: Expression) if children contains arg =>
+ val newChild = arg.asInstanceOf[Expression].transformUp(rule)
+ if (!(newChild fastEquals arg)) {
+ changed = true
+ Some(newChild)
+ } else {
+ Some(arg)
+ }
+ case m: Map[_, _] => m
+ case args: Traversable[_] => args.map {
+ case arg: Expression if children contains arg =>
+ val newChild = arg.asInstanceOf[Expression].transformUp(rule)
+ if (!(newChild fastEquals arg)) {
+ changed = true
+ newChild
+ } else {
+ arg
+ }
+ case other => other
+ }
+ case nonChild: AnyRef => nonChild
+ case null => null
+ }.toArray
+ if (changed) makeCopy(newArgs) else this
+ }
+
+ /*
+ * treeString methods
+ */
+ def nodeName = getClass.getSimpleName
+
+ def argString: String = productIterator.flatMap {
+ case e: Expression if children contains e => Nil
+ case e: Expression if e.toString contains "\n" => s"(${e.simpleString})" :: Nil
+ case seq: Seq[_] => seq.mkString("[", ",", "]") :: Nil
+ case set: Set[_] => set.mkString("{", ",", "}") :: Nil
+ case f: IDataType[_] => f.getName :: Nil
+ case other => other :: Nil
+ }.mkString(", ")
+
+ /** String representation of this node without any children */
+ def simpleString = s"$nodeName $argString"
+
+ protected def generateTreeString(depth: Int, builder: StringBuilder): StringBuilder = {
+ builder.append(" " * depth)
+ builder.append(simpleString)
+ builder.append("\n")
+ children.foreach(_.generateTreeString(depth + 1, builder))
+ builder
+ }
+
+ def treeString = generateTreeString(0, new StringBuilder).toString
+
+ /*
+ * Fluent API methods
+ */
+ def field(fieldName: String) = new UnresolvedFieldExpression(this, fieldName)
+
+ def join(fieldName: String) = field(fieldName)
+
+ def `.`(fieldName: String) = field(fieldName)
+
+ def as(alias: String) = new AliasExpression(this, alias)
+
+ def arith(op: String)(rightExpr: Expression) = new ArithmeticExpression(op, this, rightExpr)
+
+ def + = arith("+") _
+
+ def - = arith("-") _
+
+ def * = arith("*") _
+
+ def / = arith("/") _
+
+ def % = arith("%") _
+
+ def isTrait(name: String) = new isTraitUnaryExpression(name, this)
+
+ def hasField(name: String) = new hasFieldUnaryExpression(name, this)
+
+ def compareOp(op: String)(rightExpr: Expression) = new ComparisonExpression(op, this, rightExpr)
+
+ def `=` = compareOp("=") _
+
+ def `!=` = compareOp("!=") _
+
+ def `>` = compareOp(">") _
+
+ def `>=` = compareOp(">=") _
+
+ def `<` = compareOp("<") _
+
+ def `<=` = compareOp("=") _
+
+ def logicalOp(op: String)(rightExpr: Expression) = new LogicalExpression(op, List(this, rightExpr))
+
+ def and = logicalOp("and") _
+
+ def or = logicalOp("or") _
+
+ def where(condExpr: Expression) = new FilterExpression(this, condExpr)
+
+ def select(selectList: Expression*) = new SelectExpression(this, selectList.toList)
+
+ def loop(loopingExpr: Expression) = new LoopExpression(this, loopingExpr, None)
+
+ def loop(loopingExpr: Expression, times: Literal[Integer]) =
+ new LoopExpression(this, loopingExpr, Some(times))
+
+ def traitInstance() = new TraitInstanceExpression(this)
+ def instance() = new InstanceExpression(this)
+
+ def path() = new PathExpression(this)
+ }
+
+ trait BinaryNode {
+ self: Expression =>
+ def left: Expression
+
+ def right: Expression
+
+ def children = Seq(left, right)
+
+ override def namedExpressions = left.namedExpressions ++ right.namedExpressions
+ }
+
+ trait LeafNode {
+ def children = Nil
+ }
+
+ trait UnaryNode {
+ self: Expression =>
+ def child: Expression
+
+ override def namedExpressions = child.namedExpressions
+
+ def children = child :: Nil
+ }
+
+ abstract class BinaryExpression extends Expression with BinaryNode {
+ self: Product =>
+ def symbol: String
+
+ override def toString = s"($left $symbol $right)"
+ }
+
+ case class ClassExpression(clsName: String) extends Expression with LeafNode {
+ val dataType = typSystem.getDataType(classOf[ClassType], clsName)
+
+ override def toString = clsName
+ }
+
+ def _class(name: String): Expression = new ClassExpression(name)
+
+ case class TraitExpression(traitName: String) extends Expression with LeafNode {
+ val dataType = typSystem.getDataType(classOf[TraitType], traitName)
+
+ override def toString = traitName
+ }
+
+ def _trait(name: String) = new TraitExpression(name)
+
+ case class IdExpression(name: String) extends Expression with LeafNode {
+ override def toString = name
+
+ override lazy val resolved = false
+
+ override def dataType = throw new UnresolvedException(this, "id")
+ }
+
+ def id(name: String) = new IdExpression(name)
+
+ case class UnresolvedFieldExpression(child: Expression, fieldName: String) extends Expression
+ with UnaryNode {
+ override def toString = s"${child}.$fieldName"
+
+ override lazy val resolved = false
+
+ override def dataType = throw new UnresolvedException(this, "field")
+ }
+
+ case class FieldExpression(fieldName: String, fieldInfo: FieldInfo, child: Option[Expression])
+ extends Expression {
+
+ def elemType(t: IDataType[_]): IDataType[_] = {
+ if (t.getTypeCategory == TypeCategory.ARRAY) {
+ val aT = t.asInstanceOf[ArrayType]
+ if (aT.getElemType.getTypeCategory == TypeCategory.CLASS ||
+ aT.getElemType.getTypeCategory == TypeCategory.STRUCT) {
+ return aT.getElemType
+ }
+ }
+ t
+ }
+
+ val children = if (child.isDefined) List(child.get) else Nil
+ import scala.language.existentials
+ lazy val dataType = {
+ val t = {
+ if (fieldInfo.traitName != null ) {
+ typSystem.getDataType(classOf[TraitType], fieldInfo.traitName)
+ } else if (!fieldInfo.isReverse) {
+ fieldInfo.attrInfo.dataType()
+ } else {
+ fieldInfo.reverseDataType
+ }
+ }
+ elemType(t)
+ }
+ override lazy val resolved: Boolean = true
+
+ override def namedExpressions = if (child.isDefined) child.get.namedExpressions else Map()
+
+ override def toString = {
+ if (child.isDefined) {
+ val sep = if (dataType.isInstanceOf[ClassType]) " " else "."
+ s"${child.get}${sep}$fieldName"
+ } else {
+ fieldName
+ }
+ }
+ }
+
+ case class AliasExpression(child: Expression, alias: String) extends Expression with UnaryNode {
+ override def namedExpressions = child.namedExpressions + (alias -> child)
+
+ override def toString = s"$child as $alias"
+
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved child")
+ }
+ child.dataType
+ }
+ }
+
+ case class BackReference(alias: String, reference: Expression, child: Option[Expression]) extends Expression {
+ val children = if (child.isDefined) List(child.get) else Nil
+ val dataType = reference.dataType
+
+ override def namedExpressions = if (child.isDefined) child.get.namedExpressions else Map()
+
+ override def toString = if (child.isDefined) s"${child.get} $alias" else alias
+ }
+
+ case class Literal[T](dataType: PrimitiveType[T], rawValue: Any) extends Expression with LeafNode {
+ val value = if (rawValue == null) dataType.nullValue() else dataType.convert(rawValue, Multiplicity.REQUIRED)
+
+ override def toString = value match {
+ case s: String => s""""$s""""
+ case x => x.toString
+ }
+ }
+
+ def literal[T](typ: PrimitiveType[T], rawValue: Any) = new Literal[T](typ, rawValue)
+
+ def boolean(rawValue: Any) = literal(DataTypes.BOOLEAN_TYPE, rawValue)
+
+ def byte(rawValue: Any) = literal(DataTypes.BYTE_TYPE, rawValue)
+
+ def short(rawValue: Any) = literal(DataTypes.SHORT_TYPE, rawValue)
+
+ def int(rawValue: Any) = literal(DataTypes.INT_TYPE, rawValue)
+
+ def long(rawValue: Any) = literal(DataTypes.LONG_TYPE, rawValue)
+
+ def float(rawValue: Any) = literal(DataTypes.FLOAT_TYPE, rawValue)
+
+ def double(rawValue: Any) = literal(DataTypes.DOUBLE_TYPE, rawValue)
+
+ def bigint(rawValue: Any) = literal(DataTypes.BIGINTEGER_TYPE, rawValue)
+
+ def bigdecimal(rawValue: Any) = literal(DataTypes.BIGDECIMAL_TYPE, rawValue)
+
+ def string(rawValue: Any) = literal(DataTypes.STRING_TYPE, rawValue)
+
+ case class ArithmeticExpression(symbol: String,
+ left: Expression,
+ right: Expression)
+ extends BinaryExpression {
+
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved children")
+ }
+ TypeUtils.combinedType(left.dataType, right.dataType)
+ }
+ }
+
+ case class isTraitLeafExpression(traitName: String, classExpression: Option[Expression] = None)
+ extends Expression with LeafNode {
+ // validate TraitName
+ try {
+ typSystem.getDataType(classOf[TraitType], traitName)
+ } catch {
+ case me: MetadataException => throw new ExpressionException(this, "not a TraitType", me)
+ }
+
+ override lazy val resolved = classExpression.isDefined
+ lazy val dataType = {
+
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"cannot resolve isTrait application")
+ }
+
+ if (!classExpression.get.dataType.isInstanceOf[ClassType]) {
+ throw new ExpressionException(this,
+ s"Cannot apply isTrait on ${classExpression.get.dataType.getName}, it is not a ClassType")
+ }
+ DataTypes.BOOLEAN_TYPE
+ }
+
+ override def toString = s"${classExpression.getOrElse("")} is $traitName"
+ }
+
+ def isTrait(name: String) = new isTraitLeafExpression(name)
+
+ case class isTraitUnaryExpression(traitName: String, child: Expression)
+ extends Expression with UnaryNode {
+ // validate TraitName
+ typSystem.getDataType(classOf[TraitType], traitName)
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved child")
+ }
+ if (!child.dataType.isInstanceOf[ClassType]) {
+ throw new ExpressionException(this,
+ s"Cannot apply isTrait on ${child.dataType.getName}, it is not a ClassType")
+ }
+ DataTypes.BOOLEAN_TYPE
+ }
+
+ override def toString = s"$child is $traitName"
+ }
+
+ case class hasFieldLeafExpression(fieldName: String, classExpression: Option[Expression] = None)
+ extends Expression with LeafNode {
+
+ override lazy val resolved = classExpression.isDefined
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"Cannot apply hasField on ${classExpression.get.dataType.getName}, it is not a ClassType")
+ }
+ if (classExpression.isDefined && !TypeUtils.fieldMapping(classExpression.get.dataType).isDefined) {
+ throw new ExpressionException(this, s"Cannot apply hasField on ${classExpression.get.dataType.getName}")
+ }
+ DataTypes.BOOLEAN_TYPE
+ }
+
+ override def toString = s"${classExpression.getOrElse("")} has $fieldName"
+ }
+
+ def hasField(name: String) = new hasFieldLeafExpression(name)
+
+ case class hasFieldUnaryExpression(fieldName: String, child: Expression)
+ extends Expression with UnaryNode {
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved child")
+ }
+ if (!TypeUtils.fieldMapping(child.dataType).isDefined) {
+ throw new MetadataException(s"Cannot apply hasField on ${child.dataType.getName}")
+ }
+ DataTypes.BOOLEAN_TYPE
+ }
+
+ override def toString = s"$child has $fieldName"
+ }
+
+ case class ComparisonExpression(symbol: String,
+ left: Expression,
+ right: Expression)
+ extends BinaryExpression {
+
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved children")
+ }
+ if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) {
+ TypeUtils.combinedType(left.dataType, right.dataType)
+ }
+ DataTypes.BOOLEAN_TYPE
+ }
+ }
+
+ case class LogicalExpression(symbol: String, children: List[Expression])
+ extends Expression {
+ assert(children.size > 0)
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved children")
+ }
+ children.foreach { childExpr =>
+ if (childExpr.dataType != DataTypes.BOOLEAN_TYPE) {
+ throw new MetadataException(
+ s"Cannot apply logical operator '$symbol' on input of type '${childExpr.dataType}")
+ }
+ }
+ DataTypes.BOOLEAN_TYPE
+ }
+
+ override def toString = children.mkString("", s" $symbol ", "")
+ }
+
+ case class FilterExpression(val child: Expression, val condExpr: Expression) extends Expression {
+ val children = List(child, condExpr)
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved children")
+ }
+ if (condExpr.dataType != DataTypes.BOOLEAN_TYPE) {
+ throw new ExpressionException(this, s"Filter condition '$condExpr' is not a boolean expression")
+ }
+ child.dataType
+ }
+
+ override def namedExpressions = child.namedExpressions ++ condExpr.namedExpressions
+
+ override def toString = s"$child where $condExpr"
+ }
+
+ val GEN_COL_ALIAS_PREFIX = "_col"
+
+ case class SelectExpression(child: Expression, selectList: List[Expression]) extends Expression {
+
+ val children = List(child) ::: selectList
+ lazy val selectListWithAlias = selectList.zipWithIndex map {
+ case (s: AliasExpression, _) => s
+ case (x, i) => new AliasExpression(x, s"${GEN_COL_ALIAS_PREFIX}_$i")
+ }
+
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved children")
+ }
+ TypeUtils.createStructType(selectListWithAlias)
+ }
+
+ override def namedExpressions = child.namedExpressions ++ (selectList.flatMap(_.namedExpressions))
+
+ override def toString = s"""$child select ${selectListWithAlias.mkString("", ", ", "")}"""
+ }
+
+ case class LoopExpression(val input: Expression, val loopingExpression: Expression,
+ val times: Option[Literal[Integer]]) extends Expression {
+ val children = List(input, loopingExpression)
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved children")
+ }
+ if (input.dataType.getTypeCategory != TypeCategory.CLASS) {
+ throw new ExpressionException(this, s"Loop Expression applied to type : '${input.dataType.getName}';" +
+ " loop can only be applied to Class Expressions")
+ }
+ if (input.dataType != loopingExpression.dataType) {
+ throw new ExpressionException(this,
+ s"Invalid Loop Expression; input and loopExpression dataTypes don't match: " +
+ s"(${input.dataType.getName},${loopingExpression.dataType.getName}})")
+ }
+ input.dataType
+ }
+
+ override def namedExpressions = input.namedExpressions
+
+ override def toString = {
+ if (times.isDefined) s"$input loop ($loopingExpression) times ${times.get.value}"
+ else s"$input loop ($loopingExpression)"
+ }
+ }
+
+ case class TraitInstanceExpression(child: Expression)
+ extends Expression with UnaryNode {
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved child")
+ }
+ if (!child.dataType.isInstanceOf[TraitType]) {
+ throw new ExpressionException(this,
+ s"Cannot apply instance on ${child.dataType.getName}, it is not a TraitType")
+ }
+ typSystem.getIdType.getStructType
+ }
+
+ override def toString = s"$child traitInstance"
+ }
+
+ case class InstanceExpression(child: Expression)
+ extends Expression with UnaryNode {
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved child")
+ }
+ typSystem.getIdType.getStructType
+ }
+
+ override def toString = s"$child instance"
+ }
+
+ case class PathExpression(child: Expression)
+ extends Expression with UnaryNode {
+ lazy val dataType = {
+ if (!resolved) {
+ throw new UnresolvedException(this,
+ s"datatype. Can not resolve due to unresolved child")
+ }
+ TypeUtils.ResultWithPathStruct.createType(this, child.dataType)
+ }
+
+ override def toString = s"$child withPath"
+ }
+}
[11/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/atlas/typesystem/json/InstanceSerialization.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/InstanceSerialization.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/InstanceSerialization.scala
new file mode 100755
index 0000000..f792858
--- /dev/null
+++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/InstanceSerialization.scala
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json
+
+import java.text.SimpleDateFormat
+
+import org.apache.atlas.typesystem._
+import org.apache.atlas.typesystem.persistence.Id
+import org.apache.atlas.typesystem.types._
+import org.json4s._
+import org.json4s.native.Serialization._
+
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+
+object InstanceSerialization {
+
+ case class _Id(id : String, version : Int, typeName : String)
+ case class _Struct(typeName : String, values : Map[String, AnyRef])
+ case class _Reference(id : _Id,
+ typeName : String,
+ values : Map[String, AnyRef],
+ traitNames : List[String],
+ traits : Map[String, _Struct])
+
+ def Try[B](x : => B) : Option[B] = {
+ try { Some(x) } catch { case _ : Throwable => None }
+ }
+
+ /**
+ * Convert a Map into
+ * - a Reference or
+ * - a Struct or
+ * - a Id or
+ * - a Java Map whose values are recursively converted.
+ * @param jsonMap
+ * @param format
+ */
+ class InstanceJavaConversion(jsonMap : Map[String, _], format : Formats) {
+
+ /**
+ * For Id, Map must contain the [[_Id]] 'typeHint'
+ * @return
+ */
+ def idClass: Option[String] = {
+ jsonMap.get(format.typeHintFieldName).flatMap(x => Try(x.asInstanceOf[String])).
+ filter(s => s == classOf[_Id].getName)
+ }
+
+ /**
+ * validate and extract 'id' attribute from Map
+ * @return
+ */
+ def id: Option[String] = {
+ jsonMap.get("id").filter(_.isInstanceOf[String]).flatMap(v => Some(v.asInstanceOf[String]))
+ }
+
+ /**
+ * validate and extract 'version' attribute from Map
+ * @return
+ */
+ def version: Option[Int] = {
+ jsonMap.get("version").flatMap{
+ case i : Int => Some(i)
+ case bI : BigInt => Some(bI.toInt)
+ case _ => None
+ }
+ }
+
+ /**
+ * A Map is an Id if:
+ * - it has the correct [[format.typeHintFieldName]]
+ * - it has a 'typeName'
+ * - it has an 'id'
+ * - it has a 'version'
+ * @return
+ */
+ def convertId : Option[_Id] = {
+ for {
+ refClass <- idClass;
+ typNm <- typeName;
+ i <- id;
+ v <- version
+ } yield _Id(i, v, typNm)
+ }
+
+ /**
+ * validate and extract 'typeName' attribute from Map
+ * @return
+ */
+ def typeName: Option[String] = {
+ jsonMap.get("typeName").flatMap(x => Try(x.asInstanceOf[String]))
+ }
+
+ /**
+ * For Reference, Map must contain the [[_Reference]] 'typeHint'
+ * @return
+ */
+ def referenceClass: Option[String] = {
+ jsonMap.get(format.typeHintFieldName).flatMap(x => Try(x.asInstanceOf[String])).
+ filter(s => s == classOf[_Reference].getName)
+ }
+
+ /**
+ * For Reference, Map must contain the [[_Struct]] 'typeHint'
+ * @return
+ */
+ def structureClass: Option[String] = {
+ jsonMap.get(format.typeHintFieldName).flatMap(x => Try(x.asInstanceOf[String])).
+ filter(s => s == classOf[_Struct].getName)
+ }
+
+ /**
+ * validate and extract 'values' attribute from Map
+ * @return
+ */
+ def valuesMap: Option[Map[String, AnyRef]] = {
+ jsonMap.get("values").flatMap(x => Try(x.asInstanceOf[Map[String, AnyRef]]))
+ }
+
+ /**
+ * validate and extract 'traitNames' attribute from Map
+ * @return
+ */
+ def traitNames: Option[Seq[String]] = {
+ jsonMap.get("traitNames").flatMap(x => Try(x.asInstanceOf[Seq[String]]))
+ }
+
+ /**
+ * A Map is an Struct if:
+ * - it has the correct [[format.typeHintFieldName]]
+ * - it has a 'typeName'
+ * - it has a 'values' attribute
+ * @return
+ */
+ def struct: Option[_Struct] = {
+ for {
+ refClass <- structureClass;
+ typNm <- typeName;
+ values <- valuesMap
+ } yield _Struct(typNm, values)
+ }
+
+ def sequence[A](a : List[(String,Option[A])]) : Option[List[(String,A)]] = a match {
+ case Nil => Some(Nil)
+ case h :: t => {
+ h._2 flatMap {hh => sequence(t) map { (h._1,hh) :: _}}
+ }
+ }
+
+ /**
+ * Extract and convert the traits in this Map.
+ *
+ * @return
+ */
+ def traits: Option[Map[String, _Struct]] = {
+
+ /**
+ * 1. validate and extract 'traitss' attribute from Map
+ * Must be a Map[String, _]
+ */
+ val tEntry : Option[Map[String, _]] = jsonMap.get("traits").flatMap(x => Try(x.asInstanceOf[Map[String, _]]))
+
+
+ /**
+ * Try to convert each entry in traits Map into a _Struct
+ * - each entry itself must be of type Map[String, _]
+ * - recursively call InstanceJavaConversion on this Map to convert to a struct
+ */
+ val x: Option[List[(String, Option[_Struct])]] = tEntry.map { tMap: Map[String, _] =>
+ val y: Map[String, Option[_Struct]] = tMap.map { t =>
+ val tObj: Option[_Struct] = Some(t._2).flatMap(x => Try(x.asInstanceOf[Map[String, _]])).
+ flatMap { traitObj: Map[String, _] =>
+ new InstanceJavaConversion(traitObj, format).struct
+ }
+ (t._1, tObj)
+ }
+ y.toList
+ }
+
+ /**
+ * Convert a List of Optional successes into an Option of List
+ */
+ x flatMap (sequence(_)) map (_.toMap)
+
+ }
+
+ def idObject : Option[_Id] = {
+ val idM = jsonMap.get("id").flatMap(x => Try(x.asInstanceOf[Map[String, _]]))
+ idM flatMap (m => new InstanceJavaConversion(m, format).convertId)
+ }
+
+ /**
+ * A Map is an Reference if:
+ * - it has the correct [[format.typeHintFieldName]]
+ * - it has a 'typeName'
+ * - it has an _Id
+ * - it has a 'values' attribute
+ * - it has 'traitNames' attribute
+ * - it has 'traits' attribute
+ * @return
+ */
+ def reference : Option[_Reference] = {
+ for {
+ refClass <- referenceClass;
+ typNm <- typeName;
+ i <- idObject;
+ values <- valuesMap;
+ traitNms <- traitNames;
+ ts <- traits
+ } yield _Reference(i, typNm, values, traitNms.toList, ts)
+ }
+
+ /**
+ * A Map converted to Java:
+ * - if Map can be materialized as a _Reference, materialize and then recursively call asJava on it.
+ * - if Map can be materialized as a _Struct, materialize and then recursively call asJava on it.
+ * - if Map can be materialized as a _Id, materialize and then recursively call asJava on it.
+ * - otherwise convert each value with asJava and construct as new JavaMap.
+ * @return
+ */
+ def convert : Any = {
+ reference.map(asJava(_)(format)).getOrElse {
+ struct.map(asJava(_)(format)).getOrElse {
+ convertId.map(asJava(_)(format)).getOrElse {
+ jsonMap.map { t =>
+ (t._1 -> asJava(t._2)(format))
+ }.toMap.asJava
+ }
+ }
+ }
+ }
+ }
+
+ def asJava(v : Any)(implicit format: Formats) : Any = v match {
+ case i : _Id => new Id(i.id, i.version, i.typeName)
+ case s : _Struct => new Struct(s.typeName, asJava(s.values).asInstanceOf[java.util.Map[String, Object]])
+ case r : _Reference => {
+ new Referenceable(r.id.asInstanceOf[_Id].id,
+ r.typeName,
+ asJava(r.values).asInstanceOf[java.util.Map[String, Object]],
+ asJava(r.traitNames).asInstanceOf[java.util.List[String]],
+ asJava(r.traits).asInstanceOf[java.util.Map[String, IStruct]])
+ }
+ case l : List[_] => l.map(e => asJava(e)).toList.asJava
+ case m : Map[_, _] if Try{m.asInstanceOf[Map[String,_]]}.isDefined =>
+ new InstanceJavaConversion(m.asInstanceOf[Map[String,_]], format).convert
+ case _ => v
+ }
+
+ def asScala(v : Any) : Any = v match {
+ case i : Id => _Id(i._getId(), i.getVersion, i.getClassName)
+ case r : IReferenceableInstance => {
+ val traits = r.getTraits.map { tName =>
+ val t = r.getTrait(tName).asInstanceOf[IStruct]
+ (tName -> _Struct(t.getTypeName, asScala(t.getValuesMap).asInstanceOf[Map[String, AnyRef]]))
+ }.toMap
+ _Reference(asScala(r.getId).asInstanceOf[_Id],
+ r.getTypeName, asScala(r.getValuesMap).asInstanceOf[Map[String, AnyRef]],
+ asScala(r.getTraits).asInstanceOf[List[String]],
+ traits.asInstanceOf[Map[String, _Struct]])
+ }
+ case s : IStruct => _Struct(s.getTypeName, asScala(s.getValuesMap).asInstanceOf[Map[String, AnyRef]])
+ case l : java.util.List[_] => l.asScala.map(e => asScala(e)).toList
+ case m : java.util.Map[_, _] => m.asScala.map(t => (asScala(t._1), asScala(t._2))).toMap
+ case _ => v
+ }
+
+ val _formats = new DefaultFormats {
+ override val dateFormatter = TypeSystem.getInstance().getDateFormat.asInstanceOf[SimpleDateFormat]
+ override val typeHints = FullTypeHints(List(classOf[_Id], classOf[_Struct], classOf[_Reference]))
+ }
+
+ def buildFormat(withBigDecimals : Boolean) = {
+ if (withBigDecimals)
+ _formats + new BigDecimalSerializer + new BigIntegerSerializer
+ else
+ _formats
+ }
+
+ def _toJson(value: AnyRef, withBigDecimals : Boolean = false): String = {
+ implicit val formats = buildFormat(withBigDecimals)
+
+ val _s : AnyRef = asScala(value).asInstanceOf[AnyRef]
+ writePretty(_s)
+ }
+
+ def toJson(value: IStruct, withBigDecimals : Boolean = false): String = {
+ _toJson(value, withBigDecimals)
+ }
+
+ def fromJsonStruct(jsonStr: String, withBigDecimals : Boolean = false): Struct = {
+ implicit val formats = buildFormat(withBigDecimals)
+ val _s = read[_Struct](jsonStr)
+ asJava(_s).asInstanceOf[Struct]
+ }
+
+ //def toJsonReferenceable(value: Referenceable, withBigDecimals : Boolean = false): String = _toJson(value, withBigDecimals)
+ def fromJsonReferenceable(jsonStr: String, withBigDecimals : Boolean = false): Referenceable = {
+ implicit val formats = buildFormat(withBigDecimals)
+ val _s = read[_Reference](jsonStr)
+ asJava(_s).asInstanceOf[Referenceable]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
new file mode 100755
index 0000000..e38772d
--- /dev/null
+++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json
+
+import org.apache.atlas.typesystem._
+import org.apache.atlas.typesystem.persistence.{Id, ReferenceableInstance, StructInstance}
+import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory}
+import org.apache.atlas.typesystem.types._
+import org.json4s.JsonAST.JInt
+import org.json4s._
+import org.json4s.native.Serialization._
+
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+
+class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => ( {
+ case JDecimal(e) => e.bigDecimal
+}, {
+ case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
+}
+ ))
+
+class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => ( {
+ case JInt(e) => e.bigInteger
+}, {
+ case e: java.math.BigInteger => JInt(new BigInt(e))
+}
+ ))
+
+class IdSerializer extends CustomSerializer[Id](format => ( {
+ case JObject(JField("id", JInt(id)) ::
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
+ case JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("id", JInt(id)) ::
+ JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
+ case JObject(JField("id", JString(id)) ::
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("version", JString(version)) :: Nil) => new Id(id, version.toInt, className)
+ case JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("id", JString(id)) ::
+ JField("version", JString(version)) :: Nil) => new Id(id, version.toInt, className)
+}, {
+ case id: Id => JObject(JField("id", JString(id.id)),
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(id.className)),
+ JField("version", JInt(id.version)))
+}
+ ))
+
+class TypedStructSerializer() extends Serializer[ITypedStruct] {
+
+ def deserialize(implicit format: Formats) = {
+ case (TypeInfo(clazz, ptype), json) if classOf[ITypedStruct].isAssignableFrom(clazz) => json match {
+ case JObject(fs) =>
+ val (typ, fields) = fs.partition(f => f._1 == Serialization.STRUCT_TYPE_FIELD_NAME)
+ val typName = typ(0)._2.asInstanceOf[JString].s
+ val sT = typSystem.getDataType(
+ classOf[IConstructableType[IStruct, ITypedStruct]], typName).asInstanceOf[IConstructableType[IStruct, ITypedStruct]]
+ val s = sT.createInstance()
+ Serialization.deserializeFields(typSystem, sT, s, fields)
+ s
+ case x => throw new MappingException("Can't convert " + x + " to TypedStruct")
+ }
+
+ }
+
+ def typSystem = TypeSystem.getInstance()
+
+ /**
+ * Implicit conversion from `java.math.BigInteger` to `scala.BigInt`.
+ * match the builtin conversion for BigDecimal.
+ * See https://groups.google.com/forum/#!topic/scala-language/AFUamvxu68Q
+ */
+ //implicit def javaBigInteger2bigInt(x: java.math.BigInteger): BigInt = new BigInt(x)
+
+ def serialize(implicit format: Formats) = {
+ case e: ITypedStruct =>
+ val fields = Serialization.serializeFields(e)
+ JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(e.getTypeName)) :: fields)
+ }
+}
+
+class TypedReferenceableInstanceSerializer()
+ extends Serializer[ITypedReferenceableInstance] {
+
+ def deserialize(implicit format: Formats) = {
+ case (TypeInfo(clazz, ptype), json) if classOf[ITypedReferenceableInstance].isAssignableFrom(clazz) => json match {
+ case JObject(JField("id", JInt(id)) ::
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
+ case JObject(JField("id", JString(id)) ::
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("version", JInt(version)) :: Nil) => new Id(id, version.toInt, className)
+ case JObject(fs) =>
+ var typField: Option[JField] = None
+ var idField: Option[JField] = None
+ var traitsField: Option[JField] = None
+ var fields: List[JField] = Nil
+
+ fs.foreach { f: JField => f._1 match {
+ case Serialization.STRUCT_TYPE_FIELD_NAME => typField = Some(f)
+ case Serialization.ID_TYPE_FIELD_NAME => idField = Some(f)
+ case Serialization.TRAIT_TYPE_FIELD_NAME => traitsField = Some(f)
+ case _ => fields = fields :+ f
+ }
+ }
+
+ var traitNames: List[String] = Nil
+
+ traitsField.map { t =>
+ val tObj: JObject = t._2.asInstanceOf[JObject]
+ tObj.obj.foreach { oTrait =>
+ val tName: String = oTrait._1
+ traitNames = traitNames :+ tName
+ }
+ }
+
+ val typName = typField.get._2.asInstanceOf[JString].s
+ val sT = typSystem.getDataType(
+ classOf[ClassType], typName).asInstanceOf[ClassType]
+ val id = Serialization.deserializeId(idField.get._2)
+ val s = sT.createInstance(id, traitNames: _*)
+ Serialization.deserializeFields(typSystem, sT, s, fields)
+
+ traitsField.map { t =>
+ val tObj: JObject = t._2.asInstanceOf[JObject]
+ tObj.obj.foreach { oTrait =>
+ val tName: String = oTrait._1
+ val traitJObj: JObject = oTrait._2.asInstanceOf[JObject]
+ val traitObj = s.getTrait(tName).asInstanceOf[ITypedStruct]
+ val tT = typSystem.getDataType(
+ classOf[TraitType], traitObj.getTypeName).asInstanceOf[TraitType]
+ val (tTyp, tFields) = traitJObj.obj.partition(f => f._1 == Serialization.STRUCT_TYPE_FIELD_NAME)
+ Serialization.deserializeFields(typSystem, tT, traitObj, tFields)
+ }
+ }
+
+ s
+ case x => throw new MappingException("Can't convert " + x + " to TypedStruct")
+ }
+
+ }
+
+ def typSystem = TypeSystem.getInstance()
+
+ def serialize(implicit format: Formats) = {
+ case id: Id => Serialization.serializeId(id)
+ case e: ITypedReferenceableInstance =>
+ val idJ = JField(Serialization.ID_TYPE_FIELD_NAME, Serialization.serializeId(e.getId))
+ var fields = Serialization.serializeFields(e)
+ val traitsJ: List[JField] = e.getTraits.map(tName => JField(tName, Extraction.decompose(e.getTrait(tName)))).toList
+
+ fields = idJ :: fields
+ if (traitsJ.size > 0) {
+ fields = fields :+ JField(Serialization.TRAIT_TYPE_FIELD_NAME, JObject(traitsJ: _*))
+ }
+
+ JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(e.getTypeName)) :: fields)
+ }
+}
+
+
+object Serialization {
+ val STRUCT_TYPE_FIELD_NAME = "$typeName$"
+ val ID_TYPE_FIELD_NAME = "$id$"
+ val TRAIT_TYPE_FIELD_NAME = "$traits$"
+
+ def extractList(lT: ArrayType, value: JArray)(implicit format: Formats): Any = {
+ val dT = lT.getElemType
+ value.arr.map(extract(dT, _)).asJava
+ }
+
+ def extractMap(mT: MapType, value: JObject)(implicit format: Formats): Any = {
+ val kT = mT.getKeyType
+ val vT = mT.getValueType
+ value.obj.map { f: JField => f._1 -> extract(vT, f._2)}.toMap.asJava
+ }
+
+ def extract(dT: IDataType[_], value: JValue)(implicit format: Formats): Any = value match {
+ case value: JBool => Extraction.extract[Boolean](value)
+ case value: JInt => Extraction.extract[Int](value)
+ case value: JDouble => Extraction.extract[Double](value)
+ case value: JDecimal => Extraction.extract[BigDecimal](value)
+ case value: JString => Extraction.extract[String](value)
+ case JNull => null
+ case value: JArray => extractList(dT.asInstanceOf[ArrayType], value.asInstanceOf[JArray])
+ case value: JObject if dT.getTypeCategory eq TypeCategory.MAP =>
+ extractMap(dT.asInstanceOf[MapType], value.asInstanceOf[JObject])
+ case value: JObject if ((dT.getTypeCategory eq TypeCategory.STRUCT) || (dT.getTypeCategory eq TypeCategory.TRAIT)) =>
+ Extraction.extract[ITypedStruct](value)
+ case value: JObject =>
+ Extraction.extract[ITypedReferenceableInstance](value)
+ }
+
+ def serializeId(id: Id) = JObject(JField("id", JString(id.id)),
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(id.className)),
+ JField("version", JInt(id.version)))
+
+ def serializeFields(e: ITypedInstance)(implicit format: Formats) = e.fieldMapping.fields.map {
+ case (fName, info) => {
+ var v = e.get(fName)
+ if (v != null && (info.dataType().getTypeCategory eq TypeCategory.MAP)) {
+ v = v.asInstanceOf[java.util.Map[_, _]].toMap
+ }
+
+ if (v != null && (info.dataType().getTypeCategory eq TypeCategory.CLASS) && !info.isComposite) {
+ v = v.asInstanceOf[IReferenceableInstance].getId
+ }
+
+ if (v != null && (info.dataType().getTypeCategory eq TypeCategory.ENUM)) {
+ v = v.asInstanceOf[EnumValue].value
+ }
+
+ JField(fName, Extraction.decompose(v))
+ }
+ }.toList.map(_.asInstanceOf[JField])
+
+ def deserializeFields[T <: ITypedInstance](typeSystem: TypeSystem,
+ sT: IConstructableType[_, T],
+ s: T, fields: List[JField])(implicit format: Formats)
+ = {
+ //MetadataService.setCurrentService(currentMdSvc)
+ fields.foreach { f =>
+ val fName = f._1
+ val fInfo = sT.fieldMapping.fields(fName)
+ if (fInfo != null) {
+ //println(fName)
+ var v = f._2
+ if (fInfo.dataType().getTypeCategory == TypeCategory.TRAIT ||
+ fInfo.dataType().getTypeCategory == TypeCategory.STRUCT) {
+ v = v match {
+ case JObject(sFields) =>
+ JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(fInfo.dataType.getName)) :: sFields)
+ case x => x
+ }
+ }
+ s.set(fName, Serialization.extract(fInfo.dataType(), v))
+ }
+ }
+ }
+
+ def deserializeId(value: JValue)(implicit format: Formats) = value match {
+ case JObject(JField("id", JInt(id)) ::
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
+ case JObject(JField("id", JString(id)) ::
+ JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
+ JField("version", JInt(version)) :: Nil) => new Id(id, version.toInt, className)
+ }
+
+ def toJson(value: ITypedReferenceableInstance): String = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ write(value)
+ }
+
+ def toJson(value: ITypedInstance): String = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ write(value)
+ }
+
+ def toJsonPretty(value: ITypedReferenceableInstance): String = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ writePretty(value)
+ }
+
+ def fromJson(jsonStr: String): ITypedReferenceableInstance = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ read[ReferenceableInstance](jsonStr)
+ }
+
+ def traitFromJson(jsonStr: String): ITypedInstance = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ read[StructInstance](jsonStr)
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala
new file mode 100755
index 0000000..eab7806
--- /dev/null
+++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/TypesSerialization.scala
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json
+
+import java.text.SimpleDateFormat
+
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.MetadataException
+import org.apache.atlas.typesystem.TypesDef
+import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory}
+import org.apache.atlas.typesystem.types._
+import org.json4s.JsonAST.JString
+import org.json4s._
+import org.json4s.native.Serialization._
+
+
+/**
+ * Module for serializing to/from Json.
+ *
+ * @example {{{
+ * val j = TypesSerialization.toJson(typeSystem, "Employee", "Person", "Department", "SecurityClearance")
+ *
+ * val typesDef = TypesSerialization.fromJson(jsonStr)
+ * typesDef.enumTypes.foreach( typeSystem.defineEnumType(_))
+
+ typeSystem.defineTypes(ImmutableList.copyOf(typesDef.structTypes.toArray),
+ ImmutableList.copyOf(typesDef.traitTypes.toArray),
+ ImmutableList.copyOf(typesDef.classTypes.toArray)
+ )
+ * }}}
+ *
+ * @todo doesn't traverse includes directives. Includes are parsed into
+ * [[org.apache.atlas.tools.thrift.IncludeDef IncludeDef]] structures
+ * but are not traversed.
+ * @todo mixing in [[scala.util.parsing.combinator.PackratParsers PackratParsers]] is a placeholder. Need to
+ * change specific grammar rules to `lazy val` and `Parser[Elem]` to `PackratParser[Elem]`. Will do based on
+ * performance analysis.
+ * @todo Error reporting
+ */
+object TypesSerialization {
+
+ def toJsonValue(typ: IDataType[_])(implicit formats: Formats): JValue = {
+ typ.getTypeCategory match {
+ case TypeCategory.CLASS => {
+ Extraction.decompose(convertClassTypeToHierarchicalTypeDefintion(typ.asInstanceOf[ClassType]))
+ }
+ case TypeCategory.STRUCT => {
+ Extraction.decompose(convertStructTypeToStructDef(typ.asInstanceOf[StructType]))
+ }
+ case TypeCategory.TRAIT => {
+ Extraction.decompose(convertTraitTypeToHierarchicalTypeDefintion(typ.asInstanceOf[TraitType]))
+ }
+ case TypeCategory.ENUM => {
+ Extraction.decompose(convertEnumTypeToEnumTypeDef(typ.asInstanceOf[EnumType]))
+ }
+ case _ => JString(s"${typ.getName}")
+ }
+ }
+
+ def toJson(ts: TypeSystem, typName: String): String = {
+ toJson(ts, List(typName): _*)
+ }
+
+ def toJson(ts: TypeSystem, typNames: String*): String = {
+ toJson(ts, (typ: IDataType[_]) => typNames.contains(typ.getName))
+ }
+
+ import scala.collection.JavaConversions._
+
+ def toJson(ts: TypeSystem, typNames: java.util.List[String]): String = {
+ toJson(ts, typNames.toIndexedSeq: _*)
+ }
+
+ val _formats = new DefaultFormats {
+ override val dateFormatter = TypeSystem.getInstance().getDateFormat.asInstanceOf[SimpleDateFormat]
+ override val typeHints = NoTypeHints
+ }
+
+ def toJson(ts: TypeSystem, export: IDataType[_] => Boolean): String = {
+ implicit val formats = _formats + new MultiplicitySerializer
+
+ val typsDef = convertToTypesDef(ts, export)
+
+ writePretty(typsDef)
+ }
+
+ def fromJson(jsonStr: String): TypesDef = {
+ implicit val formats = _formats + new MultiplicitySerializer
+
+ read[TypesDef](jsonStr)
+ }
+
+ def toJson(typesDef : TypesDef) : String = {
+ implicit val formats = _formats + new MultiplicitySerializer
+ writePretty(typesDef)
+
+ }
+
+ def toJson(enumTypeDefinition: EnumTypeDefinition) : String = {
+ toJson(new TypesDef(enumTypeDefinition))
+ }
+
+ def toJson(structTypeDefinition: StructTypeDefinition) : String = {
+ toJson(new TypesDef(structTypeDefinition))
+ }
+
+ def toJson(typDef: HierarchicalTypeDefinition[_], isTrait : Boolean) : String = {
+ toJson(new TypesDef(typDef, isTrait))
+ }
+
+ private def convertAttributeInfoToAttributeDef(aInfo: AttributeInfo) = {
+ new AttributeDefinition(aInfo.name, aInfo.dataType().getName, aInfo.multiplicity,
+ aInfo.isComposite, aInfo.isUnique, aInfo.isIndexable, aInfo.reverseAttributeName)
+ }
+
+ private def convertEnumTypeToEnumTypeDef(et: EnumType) = {
+ val eVals: Seq[EnumValue] = et.valueMap.values().toSeq
+ new EnumTypeDefinition(et.name, eVals: _*)
+ }
+
+ private def convertStructTypeToStructDef(st: StructType): StructTypeDefinition = {
+
+ val aDefs: Iterable[AttributeDefinition] =
+ st.fieldMapping.fields.values().map(convertAttributeInfoToAttributeDef(_))
+ new StructTypeDefinition(st.name, aDefs.toArray)
+ }
+
+ private def convertTraitTypeToHierarchicalTypeDefintion(tt: TraitType): HierarchicalTypeDefinition[TraitType] = {
+
+ val aDefs: Iterable[AttributeDefinition] =
+ tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
+ new HierarchicalTypeDefinition[TraitType](classOf[TraitType], tt.name, tt.superTypes, aDefs.toArray)
+ }
+
+ private def convertClassTypeToHierarchicalTypeDefintion(tt: ClassType): HierarchicalTypeDefinition[ClassType] = {
+
+ val aDefs: Iterable[AttributeDefinition] =
+ tt.immediateAttrs.map(convertAttributeInfoToAttributeDef(_))
+ new HierarchicalTypeDefinition[ClassType](classOf[ClassType], tt.name, tt.superTypes, aDefs.toArray)
+ }
+
+ def convertToTypesDef(ts: TypeSystem, export: IDataType[_] => Boolean): TypesDef = {
+
+ var enumTypes: Seq[EnumTypeDefinition] = Nil
+ var structTypes: Seq[StructTypeDefinition] = Nil
+ var traitTypes: Seq[HierarchicalTypeDefinition[TraitType]] = Nil
+ var classTypes: Seq[HierarchicalTypeDefinition[ClassType]] = Nil
+
+ def toTyp(nm: String) = ts.getDataType(classOf[IDataType[_]], nm)
+
+ val typs: Iterable[IDataType[_]] = ts.getTypeNames.map(toTyp(_)).filter { (typ: IDataType[_]) =>
+ !(ts.getCoreTypes.contains(typ.getName)) && export(typ)
+ }
+
+ typs.foreach {
+ case typ: ArrayType => ()
+ case typ: MapType => ()
+ case typ: EnumType => enumTypes = enumTypes :+ convertEnumTypeToEnumTypeDef(typ)
+ case typ: StructType => structTypes = structTypes :+ convertStructTypeToStructDef(typ)
+ case typ: TraitType => traitTypes = traitTypes :+ convertTraitTypeToHierarchicalTypeDefintion(typ)
+ case typ: ClassType => classTypes = classTypes :+ convertClassTypeToHierarchicalTypeDefintion(typ)
+ }
+
+ TypesDef(enumTypes, structTypes, traitTypes, classTypes)
+ }
+
+}
+
+class MultiplicitySerializer extends CustomSerializer[Multiplicity](format => ( {
+ case JString(m) => m match {
+ case "optional" => Multiplicity.OPTIONAL
+ case "required" => Multiplicity.REQUIRED
+ case "collection" => Multiplicity.COLLECTION
+ case "set" => Multiplicity.SET
+ }
+}, {
+ case m: Multiplicity => JString(m match {
+ case Multiplicity.OPTIONAL => "optional"
+ case Multiplicity.REQUIRED => "required"
+ case Multiplicity.COLLECTION => "collection"
+ case Multiplicity.SET => "set"
+ }
+
+ )
+}
+ ))
+
+trait TypeHelpers {
+ def requiredAttr(name: String, dataType: IDataType[_]) =
+ new AttributeDefinition(name, dataType.getName, Multiplicity.REQUIRED, false, null)
+
+ def optionalAttr(name: String, dataTypeName: String) =
+ new AttributeDefinition(name, dataTypeName, Multiplicity.OPTIONAL, false, null)
+
+
+ def optionalAttr(name: String, dataType: IDataType[_]) =
+ new AttributeDefinition(name, dataType.getName, Multiplicity.OPTIONAL, false, null)
+
+ def structDef(name: String, attrs: AttributeDefinition*) = {
+ new StructTypeDefinition(name, attrs.toArray)
+ }
+
+ def defineTraits(ts: TypeSystem, tDefs: HierarchicalTypeDefinition[TraitType]*) = {
+ ts.defineTraitTypes(tDefs: _*)
+ }
+
+ def createTraitTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*):
+ HierarchicalTypeDefinition[TraitType] = {
+ val sts = ImmutableList.copyOf(superTypes.toArray)
+ return new HierarchicalTypeDefinition[TraitType](classOf[TraitType], name,
+ sts, attrDefs.toArray)
+ }
+
+ def createClassTypeDef(name: String, superTypes: Seq[String], attrDefs: AttributeDefinition*):
+ HierarchicalTypeDefinition[ClassType] = {
+ val sts = ImmutableList.copyOf(superTypes.toArray)
+ return new HierarchicalTypeDefinition[ClassType](classOf[ClassType], name,
+ sts, attrDefs.toArray)
+ }
+
+ @throws(classOf[MetadataException])
+ def defineClassType(ts: TypeSystem, classDef: HierarchicalTypeDefinition[ClassType]): ClassType = {
+ ts.defineTypes(ImmutableList.of[StructTypeDefinition],
+ ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
+ ImmutableList.of[HierarchicalTypeDefinition[ClassType]](classDef))
+ return ts.getDataType(classOf[ClassType], classDef.typeName)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/TypesDef.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/TypesDef.scala b/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/TypesDef.scala
deleted file mode 100755
index f96bd31..0000000
--- a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/TypesDef.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.types._
-
-import scala.collection.JavaConversions
-
-case class TypesDef(enumTypes: Seq[EnumTypeDefinition],
- structTypes: Seq[StructTypeDefinition],
- traitTypes: Seq[HierarchicalTypeDefinition[TraitType]],
- classTypes: Seq[HierarchicalTypeDefinition[ClassType]]) {
- def this() = this(Seq(), Seq(), Seq(), Seq())
- def this(enumType : EnumTypeDefinition) = this(Seq(enumType), Seq(), Seq(), Seq())
- def this(structType: StructTypeDefinition) = this(Seq(), Seq(structType), Seq(), Seq())
- def this(typ: HierarchicalTypeDefinition[_], isTrait : Boolean) = this(
- Seq(),
- Seq(),
- if ( isTrait )
- Seq(typ.asInstanceOf[HierarchicalTypeDefinition[TraitType]]) else Seq(),
- if (!isTrait )
- Seq(typ.asInstanceOf[HierarchicalTypeDefinition[ClassType]]) else Seq()
- )
-
- def enumTypesAsJavaList() = {
- import scala.collection.JavaConverters._
- enumTypes.asJava
- }
-
- def structTypesAsJavaList() = {
- import scala.collection.JavaConverters._
- structTypes.asJava
- }
-
- def traitTypesAsJavaList() = {
- import scala.collection.JavaConverters._
- traitTypes.asJava
- }
-
- def classTypesAsJavaList() = {
- import scala.collection.JavaConverters._
- classTypes.asJava
- }
-
- def isEmpty() = {
- enumTypes.isEmpty & structTypes.isEmpty & traitTypes.isEmpty & classTypes.isEmpty
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilder.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilder.scala b/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilder.scala
deleted file mode 100644
index 1367581..0000000
--- a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilder.scala
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-
-import org.apache.hadoop.metadata.typesystem.{IReferenceableInstance, IStruct, Struct, Referenceable}
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-import scala.language.dynamics
-import scala.language.implicitConversions
-import scala.util.DynamicVariable
-
-class InstanceBuilder extends Dynamic {
-
- private val references : ArrayBuffer[Referenceable] = new ArrayBuffer[Referenceable]()
-
- val context = new DynamicVariable[DynamicStruct](null)
-
- def struct(typeName : String) : DynamicStruct = {
- context.value = new DynamicStruct(this, new Struct(typeName))
- context.value
- }
-
- def instance(typeName: String, traitNames: String*)(f : => Unit) : DynamicReference = {
- val r = new Referenceable(typeName, traitNames:_*)
- references.append(r)
- val dr = new DynamicReference(this, r)
- context.withValue(dr){f}
- dr
- }
-
- def create( f : => Unit ) : java.util.List[Referenceable] = {
- f
- references.asJava
- }
-
- def applyDynamic(name : String)(value : Any) : Any = {
- context.value.updateDynamic(name)(value)
- }
-
- implicit def symbolToDynamicStruct(s : Symbol) : DynamicValue =
- new DynamicValue(this, s.name, if (context.value == null) null else context.value.s)
-
-}
-
-object DynamicValue {
-
- private[builders] def transformOut(s: IStruct, attr : String, v : Any)(implicit ib : InstanceBuilder) : DynamicValue =
- v match {
- case r : Referenceable => new DynamicReference(ib, r)
- case s : Struct => new DynamicStruct(ib, s)
- case jL : java.util.List[_] => {
- if ( s != null ) {
- new DynamicCollection(ib, attr, s)
- } else {
- new DynamicValue(ib, attr, s, jL.map{ e => transformOut(null, null, e)}.toSeq)
- }
- }
- case jM : java.util.Map[_,_] => {
- if ( s != null ) {
- new DynamicMap(ib, attr, s)
- } else {
- new DynamicValue(ib, attr, s, jM.map {
- case (k, v) => k -> transformOut(null, null, v)
- }.toMap)
- }
- }
- case x => {
- if ( s != null ) {
- new DynamicValue(ib, attr, s)
- } else {
- new DynamicValue(ib, attr, s, x)
- }
- }
- }
-
- private[builders] def transformIn(v : Any) : Any = v match {
- case dr : DynamicReference => dr.r
- case ds : DynamicStruct => ds.s
- case dv : DynamicValue => dv.get
- case l : Seq[_] => l.map{ e => transformIn(e)}.asJava
- case m : Map[_,_] => m.map {
- case (k,v) => k -> transformIn(v)
- }.asJava
- case x => x
- }
-
-}
-
-class DynamicValue(val ib : InstanceBuilder, val attrName : String, val s: IStruct, var value : Any = null) extends Dynamic {
- import DynamicValue._
-
- implicit val iib : InstanceBuilder = ib
-
- def ~(v : Any): Unit = {
- if ( s != null ) {
- s.set(attrName, transformIn(v))
- } else {
- value = v
- }
- }
-
- def get : Any = if ( s != null ) s.get(attrName) else value
-
- def selectDynamic(name: String) : DynamicValue = {
-
- throw new UnsupportedOperationException()
- }
-
- def update(key : Any, value : Object): Unit = {
- throw new UnsupportedOperationException()
- }
-
- def apply(key : Any): DynamicValue = {
-
- if ( s != null && s.isInstanceOf[Referenceable] && key.isInstanceOf[String]) {
- val r = s.asInstanceOf[Referenceable]
- if ( r.getTraits contains attrName ) {
- val traitAttr = key.asInstanceOf[String]
- return new DynamicStruct(ib, r.getTrait(attrName)).selectDynamic(traitAttr)
- }
- }
- throw new UnsupportedOperationException()
- }
-}
-
-class DynamicCollection(ib : InstanceBuilder, attrName : String, s: IStruct) extends DynamicValue(ib, attrName ,s) {
- import DynamicValue._
-
- override def update(key : Any, value : Object): Unit = {
- var jL = s.get(attrName)
- val idx = key.asInstanceOf[Int]
- if (jL == null ) {
- val l = new java.util.ArrayList[Object]()
- l.ensureCapacity(idx)
- jL = l
- }
- val nJL = new java.util.ArrayList[Object](jL.asInstanceOf[java.util.List[Object]])
- nJL.asInstanceOf[java.util.List[Object]].set(idx, transformIn(value).asInstanceOf[Object])
- s.set(attrName, nJL)
- }
-
- override def apply(key : Any): DynamicValue = {
- var jL = s.get(attrName)
- val idx = key.asInstanceOf[Int]
- if (jL == null ) {
- null
- } else {
- transformOut(null, null, jL.asInstanceOf[java.util.List[Object]].get(idx))
- }
- }
-}
-
-class DynamicMap(ib : InstanceBuilder, attrName : String, s: IStruct) extends DynamicValue(ib, attrName ,s) {
- import DynamicValue._
- override def update(key : Any, value : Object): Unit = {
- var jM = s.get(attrName)
- if (jM == null ) {
- jM = new java.util.HashMap[Object, Object]()
- }
- jM.asInstanceOf[java.util.Map[Object, Object]].put(key.asInstanceOf[AnyRef], value)
- }
-
- override def apply(key : Any): DynamicValue = {
- var jM = s.get(attrName)
- if (jM == null ) {
- null
- } else {
- transformOut(null, null, jM.asInstanceOf[java.util.Map[Object, Object]].get(key))
- }
- }
-}
-
-class DynamicStruct(ib : InstanceBuilder, s: IStruct) extends DynamicValue(ib, null ,s) {
- import DynamicValue._
- override def selectDynamic(name: String) : DynamicValue = {
- transformOut(s, name, s.get(name))
- }
-
- def updateDynamic(name: String)(value: Any) {
- s.set(name, transformIn(value))
- }
-
- override def ~(v : Any): Unit = { throw new UnsupportedOperationException()}
- override def get : Any = s
-
-}
-
-class DynamicReference(ib : InstanceBuilder, val r : IReferenceableInstance) extends DynamicStruct(ib, r) {
-
- private def _trait(name : String) = new DynamicStruct(ib, r.getTrait(name))
-
- override def selectDynamic(name: String) : DynamicValue = {
- if ( r.getTraits contains name ) {
- _trait(name)
- } else {
- super.selectDynamic(name)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilder.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilder.scala b/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilder.scala
deleted file mode 100644
index 7ac908f..0000000
--- a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilder.scala
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.TypesDef
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil
-
-import scala.collection.mutable.ArrayBuffer
-import scala.language.{dynamics, implicitConversions, postfixOps}
-import scala.util.DynamicVariable
-
-object TypesBuilder {
-
- case class Context(enums : ArrayBuffer[EnumTypeDefinition],
- structs : ArrayBuffer[StructTypeDefinition],
- classes : ArrayBuffer[HierarchicalTypeDefinition[ClassType]],
- traits : ArrayBuffer[HierarchicalTypeDefinition[TraitType]],
- currentTypeAttrs : ArrayBuffer[Attr] = null)
-
- class AttrOption()
- class ReverseAttributeName(val rNm : String) extends AttrOption
- class MultiplicityOption(val lower: Int, val upper: Int, val isUnique: Boolean) extends AttrOption
-
- val required = new AttrOption()
- val optional = new AttrOption()
- val collection = new AttrOption()
- val set = new AttrOption()
- val composite = new AttrOption()
- val unique = new AttrOption()
- val indexed = new AttrOption()
- def reverseAttributeName(rNm : String) = new ReverseAttributeName(rNm)
- def multiplicty(lower: Int, upper: Int, isUnique: Boolean) = new MultiplicityOption(lower, upper, isUnique)
-
- val boolean = DataTypes.BOOLEAN_TYPE.getName
- val byte = DataTypes.BYTE_TYPE.getName
- val short = DataTypes.SHORT_TYPE.getName
- val int = DataTypes.INT_TYPE.getName
- val long = DataTypes.LONG_TYPE.getName
- val float = DataTypes.FLOAT_TYPE.getName
-
- val double = DataTypes.DOUBLE_TYPE.getName
- val bigint = DataTypes.BIGINTEGER_TYPE.getName
- val bigdecimal = DataTypes.BIGDECIMAL_TYPE.getName
- val date = DataTypes.DATE_TYPE.getName
- val string = DataTypes.STRING_TYPE.getName
-
- def array(t : String) : String = {
- DataTypes.arrayTypeName(t)
- }
-
- def map(kt : String, vt : String) : String = {
- DataTypes.mapTypeName(kt, vt)
- }
-
- class Attr(ctx : Context, val name : String) {
-
- private var dataTypeName : String = DataTypes.BOOLEAN_TYPE.getName
- private var multiplicity: Multiplicity = Multiplicity.OPTIONAL
- private var isComposite: Boolean = false
- private var reverseAttributeName: String = null
- private var isUnique: Boolean = false
- private var isIndexable: Boolean = false
-
- ctx.currentTypeAttrs += this
-
- def getDef : AttributeDefinition =
- new AttributeDefinition(name, dataTypeName,
- multiplicity, isComposite, isUnique, isIndexable, reverseAttributeName)
-
- def `~`(dT : String, options : AttrOption*) : Attr = {
- dataTypeName = dT
- options.foreach { o =>
- o match {
- case `required` => {multiplicity = Multiplicity.REQUIRED}
- case `optional` => {multiplicity = Multiplicity.OPTIONAL}
- case `collection` => {multiplicity = Multiplicity.COLLECTION}
- case `set` => {multiplicity = Multiplicity.SET}
- case `composite` => {isComposite = true}
- case `unique` => {isUnique = true}
- case `indexed` => {isIndexable = true}
- case m : MultiplicityOption => {multiplicity = new Multiplicity(m.lower, m.upper, m.isUnique)}
- case r : ReverseAttributeName => {reverseAttributeName = r.rNm}
- case _ => ()
- }
- }
- this
- }
-
- }
-
-}
-
-class TypesBuilder {
-
- import org.apache.hadoop.metadata.typesystem.builders.TypesBuilder.{Attr, Context}
-
- val required = TypesBuilder.required
- val optional = TypesBuilder.optional
- val collection = TypesBuilder.collection
- val set = TypesBuilder.set
- val composite = TypesBuilder.composite
- val unique = TypesBuilder.unique
- val indexed = TypesBuilder.indexed
- def multiplicty = TypesBuilder.multiplicty _
- def reverseAttributeName = TypesBuilder.reverseAttributeName _
-
- val boolean = TypesBuilder.boolean
- val byte = TypesBuilder.byte
- val short = TypesBuilder.short
- val int = TypesBuilder.int
- val long = TypesBuilder.long
- val float = TypesBuilder.float
-
- val double = TypesBuilder.double
- val bigint = TypesBuilder.bigint
- val bigdecimal = TypesBuilder.bigdecimal
- val date = TypesBuilder.date
- val string = TypesBuilder.string
-
- def array = TypesBuilder.array _
-
- def map = TypesBuilder.map _
-
- val context = new DynamicVariable[Context](Context(new ArrayBuffer(),
- new ArrayBuffer(),
- new ArrayBuffer(),
- new ArrayBuffer()))
-
- implicit def strToAttr(s : String) = new Attr(context.value, s)
-
- def types(f : => Unit ) : TypesDef = {
- f
- TypesDef(context.value.enums.toSeq,
- context.value.structs.toSeq,
- context.value.traits.toSeq,
- context.value.classes.toSeq)
- }
-
- def _class(name : String, superTypes : List[String] = List())(f : => Unit): Unit = {
- val attrs = new ArrayBuffer[Attr]()
- context.withValue(context.value.copy(currentTypeAttrs = attrs)){f}
- context.value.classes +=
- TypesUtil.createClassTypeDef(name, ImmutableList.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*)
- }
-
- def _trait(name : String, superTypes : List[String] = List())(f : => Unit): Unit = {
- val attrs = new ArrayBuffer[Attr]()
- context.withValue(context.value.copy(currentTypeAttrs = attrs)){f}
- context.value.traits +=
- TypesUtil.createTraitTypeDef(name, ImmutableList.copyOf[String](superTypes.toArray), attrs.map(_.getDef):_*)
- val v = context.value
- v.traits.size
- }
-
- def struct(name : String)(f : => Unit): Unit = {
- val attrs = new ArrayBuffer[Attr]()
- context.withValue(context.value.copy(currentTypeAttrs = attrs)){f}
- context.value.structs +=
- new StructTypeDefinition(name, attrs.map(_.getDef).toArray)
- }
-
- def enum(name : String, values : String*) : Unit = {
- val enums = values.zipWithIndex.map{ case (v, i) =>
- new EnumValue(v,i)
- }
- context.value.enums +=
- TypesUtil.createEnumTypeDef(name, enums:_*)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/InstanceSerialization.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/InstanceSerialization.scala b/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/InstanceSerialization.scala
deleted file mode 100755
index 5909129..0000000
--- a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/InstanceSerialization.scala
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import java.text.SimpleDateFormat
-
-import org.apache.hadoop.metadata.typesystem._
-import org.apache.hadoop.metadata.typesystem.persistence.Id
-import org.apache.hadoop.metadata.typesystem.types._
-import org.json4s._
-import org.json4s.native.Serialization._
-
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
-
-object InstanceSerialization {
-
- case class _Id(id : String, version : Int, typeName : String)
- case class _Struct(typeName : String, values : Map[String, AnyRef])
- case class _Reference(id : _Id,
- typeName : String,
- values : Map[String, AnyRef],
- traitNames : List[String],
- traits : Map[String, _Struct])
-
- def Try[B](x : => B) : Option[B] = {
- try { Some(x) } catch { case _ : Throwable => None }
- }
-
- /**
- * Convert a Map into
- * - a Reference or
- * - a Struct or
- * - a Id or
- * - a Java Map whose values are recursively converted.
- * @param jsonMap
- * @param format
- */
- class InstanceJavaConversion(jsonMap : Map[String, _], format : Formats) {
-
- /**
- * For Id, Map must contain the [[_Id]] 'typeHint'
- * @return
- */
- def idClass: Option[String] = {
- jsonMap.get(format.typeHintFieldName).flatMap(x => Try(x.asInstanceOf[String])).
- filter(s => s == classOf[_Id].getName)
- }
-
- /**
- * validate and extract 'id' attribute from Map
- * @return
- */
- def id: Option[String] = {
- jsonMap.get("id").filter(_.isInstanceOf[String]).flatMap(v => Some(v.asInstanceOf[String]))
- }
-
- /**
- * validate and extract 'version' attribute from Map
- * @return
- */
- def version: Option[Int] = {
- jsonMap.get("version").flatMap{
- case i : Int => Some(i)
- case bI : BigInt => Some(bI.toInt)
- case _ => None
- }
- }
-
- /**
- * A Map is an Id if:
- * - it has the correct [[format.typeHintFieldName]]
- * - it has a 'typeName'
- * - it has an 'id'
- * - it has a 'version'
- * @return
- */
- def convertId : Option[_Id] = {
- for {
- refClass <- idClass;
- typNm <- typeName;
- i <- id;
- v <- version
- } yield _Id(i, v, typNm)
- }
-
- /**
- * validate and extract 'typeName' attribute from Map
- * @return
- */
- def typeName: Option[String] = {
- jsonMap.get("typeName").flatMap(x => Try(x.asInstanceOf[String]))
- }
-
- /**
- * For Reference, Map must contain the [[_Reference]] 'typeHint'
- * @return
- */
- def referenceClass: Option[String] = {
- jsonMap.get(format.typeHintFieldName).flatMap(x => Try(x.asInstanceOf[String])).
- filter(s => s == classOf[_Reference].getName)
- }
-
- /**
- * For Reference, Map must contain the [[_Struct]] 'typeHint'
- * @return
- */
- def structureClass: Option[String] = {
- jsonMap.get(format.typeHintFieldName).flatMap(x => Try(x.asInstanceOf[String])).
- filter(s => s == classOf[_Struct].getName)
- }
-
- /**
- * validate and extract 'values' attribute from Map
- * @return
- */
- def valuesMap: Option[Map[String, AnyRef]] = {
- jsonMap.get("values").flatMap(x => Try(x.asInstanceOf[Map[String, AnyRef]]))
- }
-
- /**
- * validate and extract 'traitNames' attribute from Map
- * @return
- */
- def traitNames: Option[Seq[String]] = {
- jsonMap.get("traitNames").flatMap(x => Try(x.asInstanceOf[Seq[String]]))
- }
-
- /**
- * A Map is an Struct if:
- * - it has the correct [[format.typeHintFieldName]]
- * - it has a 'typeName'
- * - it has a 'values' attribute
- * @return
- */
- def struct: Option[_Struct] = {
- for {
- refClass <- structureClass;
- typNm <- typeName;
- values <- valuesMap
- } yield _Struct(typNm, values)
- }
-
- def sequence[A](a : List[(String,Option[A])]) : Option[List[(String,A)]] = a match {
- case Nil => Some(Nil)
- case h :: t => {
- h._2 flatMap {hh => sequence(t) map { (h._1,hh) :: _}}
- }
- }
-
- /**
- * Extract and convert the traits in this Map.
- *
- * @return
- */
- def traits: Option[Map[String, _Struct]] = {
-
- /**
- * 1. validate and extract 'traitss' attribute from Map
- * Must be a Map[String, _]
- */
- val tEntry : Option[Map[String, _]] = jsonMap.get("traits").flatMap(x => Try(x.asInstanceOf[Map[String, _]]))
-
-
- /**
- * Try to convert each entry in traits Map into a _Struct
- * - each entry itself must be of type Map[String, _]
- * - recursively call InstanceJavaConversion on this Map to convert to a struct
- */
- val x: Option[List[(String, Option[_Struct])]] = tEntry.map { tMap: Map[String, _] =>
- val y: Map[String, Option[_Struct]] = tMap.map { t =>
- val tObj: Option[_Struct] = Some(t._2).flatMap(x => Try(x.asInstanceOf[Map[String, _]])).
- flatMap { traitObj: Map[String, _] =>
- new InstanceJavaConversion(traitObj, format).struct
- }
- (t._1, tObj)
- }
- y.toList
- }
-
- /**
- * Convert a List of Optional successes into an Option of List
- */
- x flatMap (sequence(_)) map (_.toMap)
-
- }
-
- def idObject : Option[_Id] = {
- val idM = jsonMap.get("id").flatMap(x => Try(x.asInstanceOf[Map[String, _]]))
- idM flatMap (m => new InstanceJavaConversion(m, format).convertId)
- }
-
- /**
- * A Map is an Reference if:
- * - it has the correct [[format.typeHintFieldName]]
- * - it has a 'typeName'
- * - it has an _Id
- * - it has a 'values' attribute
- * - it has 'traitNames' attribute
- * - it has 'traits' attribute
- * @return
- */
- def reference : Option[_Reference] = {
- for {
- refClass <- referenceClass;
- typNm <- typeName;
- i <- idObject;
- values <- valuesMap;
- traitNms <- traitNames;
- ts <- traits
- } yield _Reference(i, typNm, values, traitNms.toList, ts)
- }
-
- /**
- * A Map converted to Java:
- * - if Map can be materialized as a _Reference, materialize and then recursively call asJava on it.
- * - if Map can be materialized as a _Struct, materialize and then recursively call asJava on it.
- * - if Map can be materialized as a _Id, materialize and then recursively call asJava on it.
- * - otherwise convert each value with asJava and construct as new JavaMap.
- * @return
- */
- def convert : Any = {
- reference.map(asJava(_)(format)).getOrElse {
- struct.map(asJava(_)(format)).getOrElse {
- convertId.map(asJava(_)(format)).getOrElse {
- jsonMap.map { t =>
- (t._1 -> asJava(t._2)(format))
- }.toMap.asJava
- }
- }
- }
- }
- }
-
- def asJava(v : Any)(implicit format: Formats) : Any = v match {
- case i : _Id => new Id(i.id, i.version, i.typeName)
- case s : _Struct => new Struct(s.typeName, asJava(s.values).asInstanceOf[java.util.Map[String, Object]])
- case r : _Reference => {
- new Referenceable(r.id.asInstanceOf[_Id].id,
- r.typeName,
- asJava(r.values).asInstanceOf[java.util.Map[String, Object]],
- asJava(r.traitNames).asInstanceOf[java.util.List[String]],
- asJava(r.traits).asInstanceOf[java.util.Map[String, IStruct]])
- }
- case l : List[_] => l.map(e => asJava(e)).toList.asJava
- case m : Map[_, _] if Try{m.asInstanceOf[Map[String,_]]}.isDefined =>
- new InstanceJavaConversion(m.asInstanceOf[Map[String,_]], format).convert
- case _ => v
- }
-
- def asScala(v : Any) : Any = v match {
- case i : Id => _Id(i._getId(), i.getVersion, i.getClassName)
- case r : IReferenceableInstance => {
- val traits = r.getTraits.map { tName =>
- val t = r.getTrait(tName).asInstanceOf[IStruct]
- (tName -> _Struct(t.getTypeName, asScala(t.getValuesMap).asInstanceOf[Map[String, AnyRef]]))
- }.toMap
- _Reference(asScala(r.getId).asInstanceOf[_Id],
- r.getTypeName, asScala(r.getValuesMap).asInstanceOf[Map[String, AnyRef]],
- asScala(r.getTraits).asInstanceOf[List[String]],
- traits.asInstanceOf[Map[String, _Struct]])
- }
- case s : IStruct => _Struct(s.getTypeName, asScala(s.getValuesMap).asInstanceOf[Map[String, AnyRef]])
- case l : java.util.List[_] => l.asScala.map(e => asScala(e)).toList
- case m : java.util.Map[_, _] => m.asScala.map(t => (asScala(t._1), asScala(t._2))).toMap
- case _ => v
- }
-
- val _formats = new DefaultFormats {
- override val dateFormatter = TypeSystem.getInstance().getDateFormat.asInstanceOf[SimpleDateFormat]
- override val typeHints = FullTypeHints(List(classOf[_Id], classOf[_Struct], classOf[_Reference]))
- }
-
- def buildFormat(withBigDecimals : Boolean) = {
- if (withBigDecimals)
- _formats + new BigDecimalSerializer + new BigIntegerSerializer
- else
- _formats
- }
-
- def _toJson(value: AnyRef, withBigDecimals : Boolean = false): String = {
- implicit val formats = buildFormat(withBigDecimals)
-
- val _s : AnyRef = asScala(value).asInstanceOf[AnyRef]
- writePretty(_s)
- }
-
- def toJson(value: IStruct, withBigDecimals : Boolean = false): String = {
- _toJson(value, withBigDecimals)
- }
-
- def fromJsonStruct(jsonStr: String, withBigDecimals : Boolean = false): Struct = {
- implicit val formats = buildFormat(withBigDecimals)
- val _s = read[_Struct](jsonStr)
- asJava(_s).asInstanceOf[Struct]
- }
-
- //def toJsonReferenceable(value: Referenceable, withBigDecimals : Boolean = false): String = _toJson(value, withBigDecimals)
- def fromJsonReferenceable(jsonStr: String, withBigDecimals : Boolean = false): Referenceable = {
- implicit val formats = buildFormat(withBigDecimals)
- val _s = read[_Reference](jsonStr)
- asJava(_s).asInstanceOf[Referenceable]
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/Serialization.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/Serialization.scala b/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/Serialization.scala
deleted file mode 100755
index 7e5a634..0000000
--- a/typesystem/src/main/scala/org/apache/hadoop/metadata/typesystem/json/Serialization.scala
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import org.apache.hadoop.metadata.typesystem._
-import org.apache.hadoop.metadata.typesystem.persistence.{Id, ReferenceableInstance, StructInstance}
-import org.apache.hadoop.metadata.typesystem.types.DataTypes.{ArrayType, MapType, TypeCategory}
-import org.apache.hadoop.metadata.typesystem.types._
-import org.json4s.JsonAST.JInt
-import org.json4s._
-import org.json4s.native.Serialization._
-
-import scala.collection.JavaConversions._
-import scala.collection.JavaConverters._
-
-class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => ( {
- case JDecimal(e) => e.bigDecimal
-}, {
- case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
-}
- ))
-
-class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => ( {
- case JInt(e) => e.bigInteger
-}, {
- case e: java.math.BigInteger => JInt(new BigInt(e))
-}
- ))
-
-class IdSerializer extends CustomSerializer[Id](format => ( {
- case JObject(JField("id", JInt(id)) ::
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
- case JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("id", JInt(id)) ::
- JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
- case JObject(JField("id", JString(id)) ::
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("version", JString(version)) :: Nil) => new Id(id, version.toInt, className)
- case JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("id", JString(id)) ::
- JField("version", JString(version)) :: Nil) => new Id(id, version.toInt, className)
-}, {
- case id: Id => JObject(JField("id", JString(id.id)),
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(id.className)),
- JField("version", JInt(id.version)))
-}
- ))
-
-class TypedStructSerializer() extends Serializer[ITypedStruct] {
-
- def deserialize(implicit format: Formats) = {
- case (TypeInfo(clazz, ptype), json) if classOf[ITypedStruct].isAssignableFrom(clazz) => json match {
- case JObject(fs) =>
- val (typ, fields) = fs.partition(f => f._1 == Serialization.STRUCT_TYPE_FIELD_NAME)
- val typName = typ(0)._2.asInstanceOf[JString].s
- val sT = typSystem.getDataType(
- classOf[IConstructableType[IStruct, ITypedStruct]], typName).asInstanceOf[IConstructableType[IStruct, ITypedStruct]]
- val s = sT.createInstance()
- Serialization.deserializeFields(typSystem, sT, s, fields)
- s
- case x => throw new MappingException("Can't convert " + x + " to TypedStruct")
- }
-
- }
-
- def typSystem = TypeSystem.getInstance()
-
- /**
- * Implicit conversion from `java.math.BigInteger` to `scala.BigInt`.
- * match the builtin conversion for BigDecimal.
- * See https://groups.google.com/forum/#!topic/scala-language/AFUamvxu68Q
- */
- //implicit def javaBigInteger2bigInt(x: java.math.BigInteger): BigInt = new BigInt(x)
-
- def serialize(implicit format: Formats) = {
- case e: ITypedStruct =>
- val fields = Serialization.serializeFields(e)
- JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(e.getTypeName)) :: fields)
- }
-}
-
-class TypedReferenceableInstanceSerializer()
- extends Serializer[ITypedReferenceableInstance] {
-
- def deserialize(implicit format: Formats) = {
- case (TypeInfo(clazz, ptype), json) if classOf[ITypedReferenceableInstance].isAssignableFrom(clazz) => json match {
- case JObject(JField("id", JInt(id)) ::
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
- case JObject(JField("id", JString(id)) ::
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("version", JInt(version)) :: Nil) => new Id(id, version.toInt, className)
- case JObject(fs) =>
- var typField: Option[JField] = None
- var idField: Option[JField] = None
- var traitsField: Option[JField] = None
- var fields: List[JField] = Nil
-
- fs.foreach { f: JField => f._1 match {
- case Serialization.STRUCT_TYPE_FIELD_NAME => typField = Some(f)
- case Serialization.ID_TYPE_FIELD_NAME => idField = Some(f)
- case Serialization.TRAIT_TYPE_FIELD_NAME => traitsField = Some(f)
- case _ => fields = fields :+ f
- }
- }
-
- var traitNames: List[String] = Nil
-
- traitsField.map { t =>
- val tObj: JObject = t._2.asInstanceOf[JObject]
- tObj.obj.foreach { oTrait =>
- val tName: String = oTrait._1
- traitNames = traitNames :+ tName
- }
- }
-
- val typName = typField.get._2.asInstanceOf[JString].s
- val sT = typSystem.getDataType(
- classOf[ClassType], typName).asInstanceOf[ClassType]
- val id = Serialization.deserializeId(idField.get._2)
- val s = sT.createInstance(id, traitNames: _*)
- Serialization.deserializeFields(typSystem, sT, s, fields)
-
- traitsField.map { t =>
- val tObj: JObject = t._2.asInstanceOf[JObject]
- tObj.obj.foreach { oTrait =>
- val tName: String = oTrait._1
- val traitJObj: JObject = oTrait._2.asInstanceOf[JObject]
- val traitObj = s.getTrait(tName).asInstanceOf[ITypedStruct]
- val tT = typSystem.getDataType(
- classOf[TraitType], traitObj.getTypeName).asInstanceOf[TraitType]
- val (tTyp, tFields) = traitJObj.obj.partition(f => f._1 == Serialization.STRUCT_TYPE_FIELD_NAME)
- Serialization.deserializeFields(typSystem, tT, traitObj, tFields)
- }
- }
-
- s
- case x => throw new MappingException("Can't convert " + x + " to TypedStruct")
- }
-
- }
-
- def typSystem = TypeSystem.getInstance()
-
- def serialize(implicit format: Formats) = {
- case id: Id => Serialization.serializeId(id)
- case e: ITypedReferenceableInstance =>
- val idJ = JField(Serialization.ID_TYPE_FIELD_NAME, Serialization.serializeId(e.getId))
- var fields = Serialization.serializeFields(e)
- val traitsJ: List[JField] = e.getTraits.map(tName => JField(tName, Extraction.decompose(e.getTrait(tName)))).toList
-
- fields = idJ :: fields
- if (traitsJ.size > 0) {
- fields = fields :+ JField(Serialization.TRAIT_TYPE_FIELD_NAME, JObject(traitsJ: _*))
- }
-
- JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(e.getTypeName)) :: fields)
- }
-}
-
-
-object Serialization {
- val STRUCT_TYPE_FIELD_NAME = "$typeName$"
- val ID_TYPE_FIELD_NAME = "$id$"
- val TRAIT_TYPE_FIELD_NAME = "$traits$"
-
- def extractList(lT: ArrayType, value: JArray)(implicit format: Formats): Any = {
- val dT = lT.getElemType
- value.arr.map(extract(dT, _)).asJava
- }
-
- def extractMap(mT: MapType, value: JObject)(implicit format: Formats): Any = {
- val kT = mT.getKeyType
- val vT = mT.getValueType
- value.obj.map { f: JField => f._1 -> extract(vT, f._2)}.toMap.asJava
- }
-
- def extract(dT: IDataType[_], value: JValue)(implicit format: Formats): Any = value match {
- case value: JBool => Extraction.extract[Boolean](value)
- case value: JInt => Extraction.extract[Int](value)
- case value: JDouble => Extraction.extract[Double](value)
- case value: JDecimal => Extraction.extract[BigDecimal](value)
- case value: JString => Extraction.extract[String](value)
- case JNull => null
- case value: JArray => extractList(dT.asInstanceOf[ArrayType], value.asInstanceOf[JArray])
- case value: JObject if dT.getTypeCategory eq TypeCategory.MAP =>
- extractMap(dT.asInstanceOf[MapType], value.asInstanceOf[JObject])
- case value: JObject if ((dT.getTypeCategory eq TypeCategory.STRUCT) || (dT.getTypeCategory eq TypeCategory.TRAIT)) =>
- Extraction.extract[ITypedStruct](value)
- case value: JObject =>
- Extraction.extract[ITypedReferenceableInstance](value)
- }
-
- def serializeId(id: Id) = JObject(JField("id", JString(id.id)),
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(id.className)),
- JField("version", JInt(id.version)))
-
- def serializeFields(e: ITypedInstance)(implicit format: Formats) = e.fieldMapping.fields.map {
- case (fName, info) => {
- var v = e.get(fName)
- if (v != null && (info.dataType().getTypeCategory eq TypeCategory.MAP)) {
- v = v.asInstanceOf[java.util.Map[_, _]].toMap
- }
-
- if (v != null && (info.dataType().getTypeCategory eq TypeCategory.CLASS) && !info.isComposite) {
- v = v.asInstanceOf[IReferenceableInstance].getId
- }
-
- if (v != null && (info.dataType().getTypeCategory eq TypeCategory.ENUM)) {
- v = v.asInstanceOf[EnumValue].value
- }
-
- JField(fName, Extraction.decompose(v))
- }
- }.toList.map(_.asInstanceOf[JField])
-
- def deserializeFields[T <: ITypedInstance](typeSystem: TypeSystem,
- sT: IConstructableType[_, T],
- s: T, fields: List[JField])(implicit format: Formats)
- = {
- //MetadataService.setCurrentService(currentMdSvc)
- fields.foreach { f =>
- val fName = f._1
- val fInfo = sT.fieldMapping.fields(fName)
- if (fInfo != null) {
- //println(fName)
- var v = f._2
- if (fInfo.dataType().getTypeCategory == TypeCategory.TRAIT ||
- fInfo.dataType().getTypeCategory == TypeCategory.STRUCT) {
- v = v match {
- case JObject(sFields) =>
- JObject(JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(fInfo.dataType.getName)) :: sFields)
- case x => x
- }
- }
- s.set(fName, Serialization.extract(fInfo.dataType(), v))
- }
- }
- }
-
- def deserializeId(value: JValue)(implicit format: Formats) = value match {
- case JObject(JField("id", JInt(id)) ::
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("version", JInt(version)) :: Nil) => new Id(id.toLong, version.toInt, className)
- case JObject(JField("id", JString(id)) ::
- JField(Serialization.STRUCT_TYPE_FIELD_NAME, JString(className)) ::
- JField("version", JInt(version)) :: Nil) => new Id(id, version.toInt, className)
- }
-
- def toJson(value: ITypedReferenceableInstance): String = {
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- write(value)
- }
-
- def toJson(value: ITypedInstance): String = {
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- write(value)
- }
-
- def toJsonPretty(value: ITypedReferenceableInstance): String = {
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- writePretty(value)
- }
-
- def fromJson(jsonStr: String): ITypedReferenceableInstance = {
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- read[ReferenceableInstance](jsonStr)
- }
-
- def traitFromJson(jsonStr: String): ITypedInstance = {
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- read[StructInstance](jsonStr)
- }
-}
-
-
[27/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/Resolver.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/Resolver.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/Resolver.scala
deleted file mode 100755
index bd98d30..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/Resolver.scala
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import org.apache.hadoop.metadata.query.Expressions._
-import org.apache.hadoop.metadata.typesystem.types.IDataType
-
-class Resolver(srcExpr: Option[Expression] = None, aliases: Map[String, Expression] = Map(),
- connectClassExprToSrc: Boolean = false)
- extends PartialFunction[Expression, Expression] {
-
- import org.apache.hadoop.metadata.query.TypeUtils._
-
- def isDefinedAt(x: Expression) = true
-
- def apply(e: Expression): Expression = e match {
- case idE@IdExpression(name) => {
- val backExpr = aliases.get(name)
- if (backExpr.isDefined) {
- return new BackReference(name, backExpr.get, None)
- }
- if (srcExpr.isDefined) {
- val fInfo = resolveReference(srcExpr.get.dataType, name)
- if (fInfo.isDefined) {
- return new FieldExpression(name, fInfo.get, None)
- }
- }
- val cType = resolveAsClassType(name)
- if (cType.isDefined) {
- return new ClassExpression(name)
- }
- val tType = resolveAsTraitType(name)
- if (tType.isDefined) {
- return new TraitExpression(name)
- }
- idE
- }
- case ce@ClassExpression(clsName) if connectClassExprToSrc && srcExpr.isDefined => {
- val fInfo = resolveReference(srcExpr.get.dataType, clsName)
- if (fInfo.isDefined) {
- return new FieldExpression(clsName, fInfo.get, None)
- }
- ce
- }
- case f@UnresolvedFieldExpression(child, fieldName) if child.resolved => {
- var fInfo: Option[FieldInfo] = None
-
- fInfo = resolveReference(child.dataType, fieldName)
- if (fInfo.isDefined) {
- return new FieldExpression(fieldName, fInfo.get, Some(child))
- }
- val tType = resolveAsTraitType(fieldName)
- if (tType.isDefined) {
- return new FieldExpression(fieldName, FieldInfo(child.dataType, null, null, fieldName), Some(child))
- }
- f
- }
- case isTraitLeafExpression(traitName, classExpression)
- if srcExpr.isDefined && !classExpression.isDefined =>
- isTraitLeafExpression(traitName, srcExpr)
- case hasFieldLeafExpression(traitName, classExpression)
- if srcExpr.isDefined && !classExpression.isDefined =>
- hasFieldLeafExpression(traitName, srcExpr)
- case f@FilterExpression(inputExpr, condExpr) if inputExpr.resolved => {
- val r = new Resolver(Some(inputExpr), inputExpr.namedExpressions)
- return new FilterExpression(inputExpr, condExpr.transformUp(r))
- }
- case SelectExpression(child, selectList) if child.resolved => {
- val r = new Resolver(Some(child), child.namedExpressions)
- return new SelectExpression(child, selectList.map {
- _.transformUp(r)
- })
- }
- case l@LoopExpression(inputExpr, loopExpr, t) if inputExpr.resolved => {
- val r = new Resolver(Some(inputExpr), inputExpr.namedExpressions, true)
- return new LoopExpression(inputExpr, loopExpr.transformUp(r), t)
- }
- case x => x
- }
-}
-
-/**
- * - any FieldReferences that explicitly reference the input, can be converted to implicit references
- * - any FieldReferences that explicitly reference a
- */
-object FieldValidator extends PartialFunction[Expression, Expression] {
-
- def isDefinedAt(x: Expression) = true
-
- def isSrc(e: Expression) = e.isInstanceOf[ClassExpression] || e.isInstanceOf[TraitExpression]
-
- def validateQualifiedField(srcDataType: IDataType[_]): PartialFunction[Expression, Expression] = {
- case FieldExpression(fNm, fInfo, Some(child))
- if (child.children == Nil && !child.isInstanceOf[BackReference] && child.dataType == srcDataType) =>
- FieldExpression(fNm, fInfo, None)
- case fe@FieldExpression(fNm, fInfo, Some(child)) if isSrc(child) =>
- throw new ExpressionException(fe, s"srcType of field doesn't match input type")
- case hasFieldUnaryExpression(fNm, child) if child.dataType == srcDataType =>
- hasFieldLeafExpression(fNm)
- case hF@hasFieldUnaryExpression(fNm, child) if isSrc(child) =>
- throw new ExpressionException(hF, s"srcType of field doesn't match input type")
- case isTraitUnaryExpression(fNm, child) if child.dataType == srcDataType =>
- isTraitLeafExpression(fNm)
- case iT@isTraitUnaryExpression(fNm, child) if isSrc(child) =>
- throw new ExpressionException(iT, s"srcType of field doesn't match input type")
- }
-
- def validateOnlyFieldReferencesInLoopExpressions(loopExpr: LoopExpression)
- : PartialFunction[Expression, Unit] = {
- case f: FieldExpression => ()
- case x => throw new ExpressionException(loopExpr,
- s"Loop Expression can only contain field references; '${x.toString}' not supported.")
- }
-
- def apply(e: Expression): Expression = e match {
- case f@FilterExpression(inputExpr, condExpr) => {
- val validatedCE = condExpr.transformUp(validateQualifiedField(inputExpr.dataType))
- if (validatedCE.fastEquals(condExpr)) {
- f
- } else {
- new FilterExpression(inputExpr, validatedCE)
- }
- }
- case SelectExpression(child, selectList) if child.resolved => {
- val v = validateQualifiedField(child.dataType)
- return new SelectExpression(child, selectList.map {
- _.transformUp(v)
- })
- }
- case l@LoopExpression(inputExpr, loopExpr, t) => {
- val validatedLE = loopExpr.transformUp(validateQualifiedField(inputExpr.dataType))
- val l1 = {
- if (validatedLE.fastEquals(loopExpr)) l
- else new LoopExpression(inputExpr, validatedLE, t)
- }
- l1.loopingExpression.traverseUp(validateOnlyFieldReferencesInLoopExpressions(l1))
- l1
- }
- case x => x
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/TypeUtils.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/TypeUtils.scala
deleted file mode 100755
index a7e4a6d..0000000
--- a/repository/src/main/scala/org/apache/hadoop/metadata/query/TypeUtils.scala
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.query
-
-import java.util
-import java.util.concurrent.atomic.AtomicInteger
-
-import org.apache.hadoop.metadata.MetadataException
-import org.apache.hadoop.metadata.query.Expressions.{SelectExpression, PathExpression}
-import org.apache.hadoop.metadata.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
-import org.apache.hadoop.metadata.typesystem.types._
-
-object TypeUtils {
- val typSystem = TypeSystem.getInstance()
-
- def numericTypes : Seq[PrimitiveType[_]] = Seq(DataTypes.BYTE_TYPE,
- DataTypes.SHORT_TYPE,
- DataTypes.INT_TYPE,
- DataTypes.FLOAT_TYPE,
- DataTypes.LONG_TYPE,
- DataTypes.DOUBLE_TYPE,
- DataTypes.BIGINTEGER_TYPE,
- DataTypes.BIGDECIMAL_TYPE)
-
- def combinedType(typ1 : IDataType[_], typ2 : IDataType[_]) : PrimitiveType[_] = {
- val typ1Idx = if (numericTypes.contains(typ1)) Some(numericTypes.indexOf(typ1)) else None
- val typ2Idx = if (numericTypes.contains(typ2)) Some(numericTypes.indexOf(typ2)) else None
-
- if ( typ1Idx.isDefined && typ2Idx.isDefined ) {
- val rIdx = math.max(typ1Idx.get, typ2Idx.get)
-
- if ( (typ1 == DataTypes.FLOAT_TYPE && typ2 == DataTypes.LONG_TYPE) ||
- (typ1 == DataTypes.LONG_TYPE && typ2 == DataTypes.FLOAT_TYPE) ) {
- return DataTypes.DOUBLE_TYPE
- }
- return numericTypes(rIdx)
- }
-
- throw new MetadataException(s"Cannot combine types: ${typ1.getName} and ${typ2.getName}")
- }
-
- var tempStructCounter : AtomicInteger = new AtomicInteger(0)
- val TEMP_STRUCT_NAME_PREFIX = "__tempQueryResultStruct"
- def createStructType(selectExprs : List[Expressions.AliasExpression]) : StructType = {
- val aDefs = new Array[AttributeDefinition](selectExprs.size)
- selectExprs.zipWithIndex.foreach { t =>
- val (e,i) = t
- aDefs(i) = new AttributeDefinition(e.alias,e.dataType.getName, Multiplicity.OPTIONAL, false, null)
- }
- return typSystem.defineQueryResultType(s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}",
- null,
- aDefs:_*);
- }
-
- object ResultWithPathStruct {
- val pathAttrName = "path"
- val resultAttrName = "result"
- val pathAttrType = DataTypes.arrayTypeName(typSystem.getIdType.getStructType)
-
- val pathAttr = new AttributeDefinition(pathAttrName, pathAttrType, Multiplicity.COLLECTION, false, null)
-
- def createType(pE : PathExpression, resultType : IDataType[_]) : StructType = {
- val resultAttr = new AttributeDefinition(resultAttrName, resultType.getName, Multiplicity.REQUIRED, false, null)
- val typName = s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}"
- val m : java.util.HashMap[String, IDataType[_]] = new util.HashMap[String, IDataType[_]]()
- if ( pE.child.isInstanceOf[SelectExpression]) {
- m.put(pE.child.dataType.getName, pE.child.dataType)
- }
- typSystem.defineQueryResultType(typName, m, pathAttr, resultAttr);
- }
- }
-
- /**
- * Structure representing the Closure Graph.
- * Returns:
- * 1. A map of vertexId -> vertex Info(these are the attributes requested in the query)
- * 2. A edges map: each entry is a mapping from an vertexId to the List of adjacent vertexIds.
- *
- * '''The Vertex Map doesn't contain all the vertices in the Graph. Only the ones for which Attributes are
- * available.''' These are the vertices that represent the EntityType whose Closure was requested. For e.g. for
- * Table Lineage the ''vertex map'' will contain information about Tables, but not about ''Load Process'' vertices
- * that connect Tables.
- */
- object GraphResultStruct {
- val SRC_PREFIX = "src"
- val DEST_PREFIX = "dest"
-
- val verticesAttrName = "vertices"
- val edgesAttrName = "edges"
- val vertexIdAttrName = "vertexId"
-
- lazy val edgesAttrType = typSystem.defineMapType(DataTypes.STRING_TYPE,
- typSystem.defineArrayType(DataTypes.STRING_TYPE))
-
- def createType(resultWithPathType: StructType): StructType = {
- val resultType = resultWithPathType.fieldMapping().fields.get(ResultWithPathStruct.resultAttrName).dataType()
-
- val verticesAttrType = typSystem.defineMapType(DataTypes.STRING_TYPE,
- vertexType(resultType.asInstanceOf[StructType]))
- val typName = s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}"
- val verticesAttr = new AttributeDefinition(verticesAttrName, verticesAttrType.getName,
- Multiplicity.REQUIRED, false, null)
- val edgesAttr = new AttributeDefinition(edgesAttrName, edgesAttrType.getName, Multiplicity.REQUIRED, false, null)
-
- val m: java.util.HashMap[String, IDataType[_]] = new util.HashMap[String, IDataType[_]]()
- m.put(resultWithPathType.getName, resultWithPathType)
- m.put(resultType.getName, resultType)
- m.put(edgesAttrType.getName, edgesAttrType)
- m.put(verticesAttrType.getName, verticesAttrType)
- typSystem.defineQueryResultType(typName, m, verticesAttr, edgesAttr)
- }
-
- private def vertexType(resultType: StructType): StructType = {
-
- import scala.collection.JavaConverters._
-
- var attrs: List[AttributeDefinition] =
- resultType.fieldMapping.fields.asScala.filter(_._1.startsWith(s"${SRC_PREFIX}_")).mapValues { aInfo =>
-
- new AttributeDefinition(aInfo.name.substring(s"${SRC_PREFIX}_".length), aInfo.dataType.getName,
- aInfo.multiplicity, aInfo.isComposite, aInfo.reverseAttributeName)
- }.values.toList
-
- attrs = new AttributeDefinition(vertexIdAttrName, typSystem.getIdType.getStructType.name,
- Multiplicity.REQUIRED, false, null) :: attrs
-
- return typSystem.defineQueryResultType(s"${TEMP_STRUCT_NAME_PREFIX}${tempStructCounter.getAndIncrement}",
- null,
- attrs: _*)
- }
- }
-
- def fieldMapping(iDataType: IDataType[_]) : Option[FieldMapping] = iDataType match {
- case c : ClassType => Some(c.fieldMapping())
- case t : TraitType => Some(t.fieldMapping())
- case s : StructType => Some(s.fieldMapping())
- case _ => None
- }
-
- def hasFields(iDataType: IDataType[_]) : Boolean = {
- fieldMapping(iDataType).isDefined
- }
-
- import scala.language.existentials
- case class FieldInfo(dataType : IDataType[_],
- attrInfo : AttributeInfo,
- reverseDataType : IDataType[_] = null,
- traitName : String = null) {
- def isReverse = reverseDataType != null
- override def toString : String = {
- if ( traitName != null ) {
- s"""FieldInfo("${dataType.getName}", "$traitName")"""
- }
- else if ( reverseDataType == null ) {
- s"""FieldInfo("${dataType.getName}", "${attrInfo.name}")"""
- } else {
- s"""FieldInfo("${dataType.getName}", "${attrInfo.name}", "${reverseDataType.getName}")"""
- }
- }
- }
-
- val FIELD_QUALIFIER = "(.*?)(->.*)?".r
-
- /**
- * Given a ComposedType `t` and a name resolve using the following rules:
- * - if `id` is a field in `t` resolve to the field
- * - if `id` is the name of a Struct|Class|Trait Type and it has a field that is of type `t` then return that type
- *
- * For e.g.
- * 1. if we have types Table(name : String, cols : List[Column]), Column(name : String) then
- * `resolveReference(Table, "cols")` resolves to type Column. So a query can be "Table.cols"
- * 2. But if we have Table(name : String), Column(name : String, tbl : Table) then "Table.Column" will resolve
- * to type Column
- *
- * This way the language will support navigation even if the relationship is one-sided.
- *
- * @param typ
- * @param id
- * @return
- */
- def resolveReference(typ : IDataType[_], id : String) : Option[FieldInfo] = {
-
- val fMap = fieldMapping(typ)
- if ( fMap.isDefined) {
-
- if (fMap.get.fields.containsKey(id)) {
- return Some(FieldInfo(typ,fMap.get.fields.get(id)))
- }
-
- try {
- val FIELD_QUALIFIER(clsNm, rest) = id
- val idTyp = typSystem.getDataType(classOf[IDataType[_]], clsNm)
- val idTypFMap = fieldMapping(idTyp)
-
- if (rest != null ) {
- val attrNm = rest.substring(2)
-
- if (idTypFMap.get.fields.containsKey(attrNm)) {
- return Some(FieldInfo(typ,idTypFMap.get.fields.get(attrNm), idTyp))
- }
- }
-
- if (idTypFMap.isDefined) {
- import scala.collection.JavaConversions._
- val fields: Seq[AttributeInfo] = idTypFMap.get.fields.values().filter { aInfo =>
- aInfo.dataType() == typ ||
- ( aInfo.dataType().getTypeCategory == TypeCategory.ARRAY &&
- aInfo.dataType().asInstanceOf[ArrayType].getElemType == typ
- )
- }.toSeq
- if (fields.size == 1) {
- return Some(FieldInfo(typ, fields(0), idTyp))
- }
- /*
- * is there only 1 array field of this type?
- * If yes resolve to it.
- * @todo: allow user to specify the relationship to follow by further qualifying the type. for e.g.
- * field("LoadProcess.inputTables")
- */
- val aFields = fields.filter { aInfo => aInfo.dataType().getTypeCategory == TypeCategory.ARRAY}
- if (aFields.size == 1) {
- return Some(FieldInfo(typ, aFields(0), idTyp))
- }
- }
- } catch {
- case _ : MetadataException => None
- }
- }
- None
- }
-
- def resolveAsClassType(id : String) : Option[ClassType] = {
- try {
- Some(typSystem.getDataType(classOf[ClassType], id))
- } catch {
- case _ : MetadataException => None
- }
- }
-
- def resolveAsTraitType(id : String) : Option[TraitType] = {
- try {
- Some(typSystem.getDataType(classOf[TraitType], id))
- } catch {
- case _ : MetadataException => None
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
new file mode 100755
index 0000000..4195955
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.testng.Assert;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+
+/**
+ * Unit test for Guice injector service loading
+ *
+ * Uses TestNG's Guice annotation to load the necessary modules and inject the
+ * objects from Guice
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class RepositoryServiceLoadingTest {
+
+ @Inject
+ private GraphProvider<TitanGraph> graphProvider;
+
+ @Test
+ public void testGetGraphService() throws Exception {
+ Assert.assertNotNull(graphProvider);
+ Assert.assertNotNull(graphProvider.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
new file mode 100755
index 0000000..c7f35fc
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import com.google.common.collect.ImmutableList;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.testng.Assert;
+
+import java.io.File;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
+
+/**
+ * Test utility class.
+ */
+public final class TestUtils {
+
+ private TestUtils() {
+ }
+
+ /**
+ * Dumps the graph in GSON format in the path returned.
+ *
+ * @param titanGraph handle to graph
+ * @return path to the dump file
+ * @throws Exception
+ */
+ public static String dumpGraph(TitanGraph titanGraph) throws Exception {
+ File tempFile = File.createTempFile("graph", ".gson");
+ System.out.println("tempFile.getPath() = " + tempFile.getPath());
+ GraphSONWriter.outputGraph(titanGraph, tempFile.getPath());
+
+ System.out.println("Vertices:");
+ for (Vertex vertex : titanGraph.getVertices()) {
+ System.out.println(GraphHelper.vertexString(vertex));
+ }
+
+ System.out.println("Edges:");
+ for (Edge edge : titanGraph.getEdges()) {
+ System.out.println(GraphHelper.edgeString(edge));
+ }
+
+ return tempFile.getPath();
+ }
+
+ /**
+ * Class Hierarchy is:
+ * Department(name : String, employees : Array[Person])
+ * Person(name : String, department : Department, manager : Manager)
+ * Manager(subordinates : Array[Person]) extends Person
+ * <p/>
+ * Persons can have SecurityClearance(level : Int) clearance.
+ */
+ public static void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
+
+ EnumTypeDefinition orgLevelEnum =
+ new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1), new EnumValue("L2", 2));
+ ts.defineEnumType(orgLevelEnum);
+
+ StructTypeDefinition addressDetails = createStructTypeDef("Address",
+ createRequiredAttrDef("street", DataTypes.STRING_TYPE),
+ createRequiredAttrDef("city", DataTypes.STRING_TYPE));
+
+ HierarchicalTypeDefinition<ClassType> deptTypeDef =
+ createClassTypeDef("Department", ImmutableList.<String>of(),
+ createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees",
+ String.format("array<%s>", "Person"), Multiplicity.COLLECTION, true,
+ "department")
+ );
+
+ HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person",
+ ImmutableList.<String>of(),
+ createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ createOptionalAttrDef("orgLevel", ts.getDataType(EnumType.class, "OrgLevel")),
+ createOptionalAttrDef("address", "Address"),
+ new AttributeDefinition("department",
+ "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager",
+ "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ );
+
+ HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager",
+ ImmutableList.of("Person"),
+ new AttributeDefinition("subordinates",
+ String.format("array<%s>", "Person"), Multiplicity.COLLECTION, false,
+ "manager")
+ );
+
+ HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef = createTraitTypeDef(
+ "SecurityClearance",
+ ImmutableList.<String>of(),
+ createRequiredAttrDef("level", DataTypes.INT_TYPE)
+ );
+
+ ts.defineTypes(ImmutableList.of(addressDetails),
+ ImmutableList.of(securityClearanceTypeDef),
+ ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef));
+ }
+
+ public static Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
+ Referenceable hrDept = new Referenceable("Department");
+ Referenceable john = new Referenceable("Person");
+ Referenceable jane = new Referenceable("Manager", "SecurityClearance");
+ Referenceable johnAddr = new Referenceable("Address");
+ Referenceable janeAddr = new Referenceable("Address");
+
+ hrDept.set("name", "hr");
+ john.set("name", "John");
+ john.set("department", hrDept);
+ johnAddr.set("street", "Stewart Drive");
+ johnAddr.set("city", "Sunnyvale");
+ john.set("address", johnAddr);
+
+ jane.set("name", "Jane");
+ jane.set("department", hrDept);
+ janeAddr.set("street", "Great America Parkway");
+ janeAddr.set("city", "Santa Clara");
+ jane.set("address", janeAddr);
+
+ john.set("manager", jane);
+
+ hrDept.set("employees", ImmutableList.of(john, jane));
+
+ jane.set("subordinates", ImmutableList.of(john));
+
+ jane.getTrait("SecurityClearance").set("level", 1);
+
+ ClassType deptType = ts.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+ Assert.assertNotNull(hrDept2);
+
+ return hrDept;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
new file mode 100755
index 0000000..4d12ccd
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import com.google.common.collect.ImmutableList;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.query.HiveTitanSample;
+import org.apache.atlas.query.QueryTestsUtils;
+import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.commons.io.FileUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import javax.script.Bindings;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import java.io.File;
+
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+
+@Guice(modules = RepositoryMetadataModule.class)
+public class GraphBackedDiscoveryServiceTest {
+
+ @Inject
+ private GraphProvider<TitanGraph> graphProvider;
+
+ @Inject
+ private GraphBackedMetadataRepository repositoryService;
+
+ @Inject
+ private GraphBackedDiscoveryService discoveryService;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ TypeSystem typeSystem = TypeSystem.getInstance();
+ typeSystem.reset();
+
+ QueryTestsUtils.setupTypes();
+ setupSampleData();
+
+ TestUtils.defineDeptEmployeeTypes(typeSystem);
+
+ Referenceable hrDept = TestUtils.createDeptEg1(typeSystem);
+ ClassType deptType = typeSystem.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+ repositoryService.createEntity(hrDept2);
+ }
+
+ private void setupSampleData() throws ScriptException {
+ TitanGraph titanGraph = graphProvider.get();
+
+ ScriptEngineManager manager = new ScriptEngineManager();
+ ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
+ Bindings bindings = engine.createBindings();
+ bindings.put("g", titanGraph);
+
+ String hiveGraphFile = FileUtils.getTempDirectory().getPath()
+ + File.separator + System.nanoTime() + ".gson";
+ System.out.println("hiveGraphFile = " + hiveGraphFile);
+ HiveTitanSample.writeGson(hiveGraphFile);
+ bindings.put("hiveGraphFile", hiveGraphFile);
+
+ engine.eval("g.loadGraphSON(hiveGraphFile)", bindings);
+ titanGraph.commit();
+
+ System.out.println("*******************Graph Dump****************************");
+ for (Vertex vertex : titanGraph.getVertices()) {
+ System.out.println(GraphHelper.vertexString(vertex));
+ }
+
+ for (Edge edge : titanGraph.getEdges()) {
+ System.out.println(GraphHelper.edgeString(edge));
+ }
+ System.out.println("*******************Graph Dump****************************");
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ TypeSystem.getInstance().reset();
+ }
+
+ @Test
+ public void testSearchByDSL() throws Exception {
+ String dslQuery = "from Department";
+
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ Assert.assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ Assert.assertEquals(results.length(), 3);
+ System.out.println("results = " + results);
+
+ Object query = results.get("query");
+ Assert.assertNotNull(query);
+
+ JSONObject dataType = results.getJSONObject("dataType");
+ Assert.assertNotNull(dataType);
+ String typeName = dataType.getString("typeName");
+ Assert.assertNotNull(typeName);
+ Assert.assertEquals(typeName, "Department");
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertNotNull(rows);
+ Assert.assertEquals(rows.length(), 1);
+ }
+
+ @Test(expectedExceptions = Throwable.class)
+ public void testSearchByDSLBadQuery() throws Exception {
+ String dslQuery = "from blah";
+
+ discoveryService.searchByDSL(dslQuery);
+ Assert.fail();
+ }
+
+ @Test
+ public void testRawSearch1() throws Exception {
+ // Query for all Vertices in Graph
+ Object r = discoveryService.searchByGremlin("g.V.toList()");
+ System.out.println("search result = " + r);
+
+ // Query for all Vertices of a Type
+ r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Department'}.toList()");
+ System.out.println("search result = " + r);
+
+ // Property Query: list all Person names
+ r = discoveryService
+ .searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
+ System.out.println("search result = " + r);
+ }
+
+ @DataProvider(name = "dslQueriesProvider")
+ private Object[][] createDSLQueries() {
+ return new String[][] {
+ {"from DB"},
+ {"DB"},
+ {"DB where DB.name=\"Reporting\""},
+ {"DB DB.name = \"Reporting\""},
+ {"DB where DB.name=\"Reporting\" select name, owner"},
+ {"DB has name"},
+ {"DB, Table"},
+ {"DB is JdbcAccess"},
+ /*
+ {"DB, LoadProcess has name"},
+ {"DB as db1, Table where db1.name = \"Reporting\""},
+ {"DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
+ */
+ {"from Table"},
+ {"Table"},
+ {"Table is Dimension"},
+ {"Column where Column isa PII"},
+ {"View is Dimension"},
+ /*{"Column where Column isa PII select Column.name"},*/
+ {"Column select Column.name"},
+ {"Column select name"},
+ {"Column where Column.name=\"customer_id\""},
+ {"from Table select Table.name"},
+ {"DB where (name = \"Reporting\")"},
+ {"DB where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
+ {"DB where DB is JdbcAccess"},
+ {"DB where DB has name"},
+ {"DB Table"},
+ {"DB where DB has name"},
+ {"DB as db1 Table where (db1.name = \"Reporting\")"},
+ {"DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
+ /*
+ todo: does not work
+ {"DB where (name = \"Reporting\") and ((createTime + 1) > 0)"},
+ {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
+ {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
+ {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
+ {"DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
+ */
+ // trait searches
+ {"Dimension"},
+ /*{"Fact"}, - todo: does not work*/
+ {"JdbcAccess"},
+ {"ETL"},
+ {"Metric"},
+ {"PII"},
+ // Lineage
+ {"Table LoadProcess outputTable"},
+ {"Table loop (LoadProcess outputTable)"},
+ {"Table as _loop0 loop (LoadProcess outputTable) withPath"},
+ {"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable withPath"},
+ {"Table as t, sd, Column as c where t.name=\"sales_fact\" select c.name as colName, c.dataType as colType"},
+ {"Table where name='sales_fact', db where name='Reporting'"}
+ };
+ }
+
+ @Test (dataProvider = "dslQueriesProvider")
+ public void testSearchByDSLQueries(String dslQuery) throws Exception {
+ System.out.println("Executing dslQuery = " + dslQuery);
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ Assert.assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ Assert.assertEquals(results.length(), 3);
+ System.out.println("results = " + results);
+
+ Object query = results.get("query");
+ Assert.assertNotNull(query);
+
+ JSONObject dataType = results.getJSONObject("dataType");
+ Assert.assertNotNull(dataType);
+ String typeName = dataType.getString("typeName");
+ Assert.assertNotNull(typeName);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertNotNull(rows);
+ Assert.assertTrue(rows.length() >= 0); // some queries may not have any results
+ System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
+ }
+
+ @DataProvider(name = "invalidDslQueriesProvider")
+ private Object[][] createInvalidDSLQueries() {
+ return new String[][] {
+ {"from Unknown"},
+ {"Unknown"},
+ {"Unknown is Blah"},
+ };
+ }
+
+ @Test (dataProvider = "invalidDslQueriesProvider", expectedExceptions = DiscoveryException.class)
+ public void testSearchByDSLInvalidQueries(String dslQuery) throws Exception {
+ System.out.println("Executing dslQuery = " + dslQuery);
+ discoveryService.searchByDSL(dslQuery);
+ Assert.fail();
+ }
+
+ @Test
+ public void testSearchForTypeInheritance() throws Exception {
+ createTypesWithMultiLevelInheritance();
+ createInstances();
+
+ String dslQuery = "from D where a = 1";
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ Assert.assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ System.out.println("results = " + results);
+ }
+
+ /*
+ * Type Hierarchy is:
+ * A(a)
+ * B(b) extends A
+ * C(c) extends B
+ * D(d) extends C
+ */
+ private void createTypesWithMultiLevelInheritance() throws Exception {
+ HierarchicalTypeDefinition A = createClassTypeDef("A", null,
+ createRequiredAttrDef("a", DataTypes.INT_TYPE));
+
+ HierarchicalTypeDefinition B = createClassTypeDef("B", ImmutableList.of("A"),
+ createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE));
+
+ HierarchicalTypeDefinition C = createClassTypeDef("C", ImmutableList.of("B"),
+ createOptionalAttrDef("c", DataTypes.BYTE_TYPE));
+
+ HierarchicalTypeDefinition D = createClassTypeDef("D", ImmutableList.of("C"),
+ createOptionalAttrDef("d", DataTypes.SHORT_TYPE));
+
+ TypeSystem.getInstance().defineClassTypes(A, B, C, D);
+ }
+
+ private void createInstances() throws Exception {
+ Referenceable instance = new Referenceable("D");
+ instance.set("d", 1);
+ instance.set("c", 1);
+ instance.set("b", true);
+ instance.set("a", 1);
+
+ ClassType deptType = TypeSystem.getInstance().getDataType(ClassType.class, "D");
+ ITypedReferenceableInstance typedInstance =
+ deptType.convert(instance, Multiplicity.REQUIRED);
+
+ repositoryService.createEntity(typedInstance);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
new file mode 100644
index 0000000..9141d85
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
@@ -0,0 +1,593 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.services.DefaultMetadataService;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.List;
+
+/**
+ * Unit tests for Hive LineageService.
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class HiveLineageServiceTest {
+
+ @Inject
+ private DefaultMetadataService metadataService;
+
+ @Inject
+ private GraphBackedDiscoveryService discoveryService;
+
+ @Inject
+ private HiveLineageService hiveLineageService;
+
+// @Inject
+// private GraphProvider<TitanGraph> graphProvider;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ setUpTypes();
+ setupInstances();
+
+ // TestUtils.dumpGraph(graphProvider.get());
+ }
+
+ @DataProvider(name = "dslQueriesProvider")
+ private Object[][] createDSLQueries() {
+ return new String[][] {
+ // joins
+ {"hive_table where name=\"sales_fact\", columns"},
+ {"hive_table where name=\"sales_fact\", columns select name, dataType, comment"},
+ {"hive_table where name=\"sales_fact\", columns as c select c.name, c.dataType, c.comment"},
+// {"hive_db as db where (db.name=\"Reporting\"), hive_table as table select db.name, table.name"},
+ {"from hive_db"},
+ {"hive_db"},
+ {"hive_db where hive_db.name=\"Reporting\""},
+ {"hive_db hive_db.name = \"Reporting\""},
+ {"hive_db where hive_db.name=\"Reporting\" select name, owner"},
+ {"hive_db has name"},
+// {"hive_db, hive_table"},
+// {"hive_db, hive_process has name"},
+// {"hive_db as db1, hive_table where db1.name = \"Reporting\""},
+// {"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System.currentTimeMillis()},
+ {"from hive_table"},
+ {"hive_table"},
+ {"hive_table is Dimension"},
+ {"hive_column where hive_column isa PII"},
+// {"hive_column where hive_column isa PII select hive_column.name"},
+ {"hive_column select hive_column.name"},
+ {"hive_column select name"},
+ {"hive_column where hive_column.name=\"customer_id\""},
+ {"from hive_table select hive_table.name"},
+ {"hive_db where (name = \"Reporting\")"},
+ {"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1"},
+ {"hive_db where hive_db has name"},
+// {"hive_db hive_table"},
+ {"hive_db where hive_db has name"},
+// {"hive_db as db1 hive_table where (db1.name = \"Reporting\")"},
+ {"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 "},
+// {"hive_db where (name = \"Reporting\") and ((createTime + 1) > 0)"},
+// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
+// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName"},
+// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
+// {"hive_db as db1 hive_table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName"},
+ // trait searches
+ {"Dimension"},
+ {"Fact"},
+ {"ETL"},
+ {"Metric"},
+ {"PII"},
+ };
+ }
+
+ @Test (dataProvider = "dslQueriesProvider")
+ public void testSearchByDSLQueries(String dslQuery) throws Exception {
+ System.out.println("Executing dslQuery = " + dslQuery);
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ Assert.assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ Assert.assertEquals(results.length(), 3);
+ System.out.println("results = " + results);
+
+ Object query = results.get("query");
+ Assert.assertNotNull(query);
+
+ JSONObject dataType = results.getJSONObject("dataType");
+ Assert.assertNotNull(dataType);
+ String typeName = dataType.getString("typeName");
+ Assert.assertNotNull(typeName);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertNotNull(rows);
+ Assert.assertTrue(rows.length() >= 0); // some queries may not have any results
+ System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
+ }
+
+ @Test
+ public void testGetInputs() throws Exception {
+ JSONObject results = new JSONObject(hiveLineageService.getInputs("sales_fact_monthly_mv"));
+ Assert.assertNotNull(results);
+ System.out.println("inputs = " + results);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertTrue(rows.length() > 0);
+
+ final JSONObject row = rows.getJSONObject(0);
+ JSONArray paths = row.getJSONArray("path");
+ Assert.assertTrue(paths.length() > 0);
+ }
+
+ @Test (expectedExceptions = IllegalArgumentException.class)
+ public void testGetInputsTableNameNull() throws Exception {
+ hiveLineageService.getInputs(null);
+ Assert.fail();
+ }
+
+ @Test (expectedExceptions = IllegalArgumentException.class)
+ public void testGetInputsTableNameEmpty() throws Exception {
+ hiveLineageService.getInputs("");
+ Assert.fail();
+ }
+
+ @Test (expectedExceptions = EntityNotFoundException.class)
+ public void testGetInputsBadTableName() throws Exception {
+ hiveLineageService.getInputs("blah");
+ Assert.fail();
+ }
+
+ @Test
+ public void testGetInputsGraph() throws Exception {
+ JSONObject results = new JSONObject(
+ hiveLineageService.getInputsGraph("sales_fact_monthly_mv"));
+ Assert.assertNotNull(results);
+ System.out.println("inputs graph = " + results);
+
+ JSONObject values = results.getJSONObject("values");
+ Assert.assertNotNull(values);
+
+ final JSONObject vertices = values.getJSONObject("vertices");
+ Assert.assertEquals(vertices.length(), 4);
+
+ final JSONObject edges = values.getJSONObject("edges");
+ Assert.assertEquals(edges.length(), 4);
+ }
+
+ @Test
+ public void testGetOutputs() throws Exception {
+ JSONObject results = new JSONObject(hiveLineageService.getOutputs("sales_fact"));
+ Assert.assertNotNull(results);
+ System.out.println("outputs = " + results);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertTrue(rows.length() > 0);
+
+ final JSONObject row = rows.getJSONObject(0);
+ JSONArray paths = row.getJSONArray("path");
+ Assert.assertTrue(paths.length() > 0);
+ }
+
+ @Test (expectedExceptions = IllegalArgumentException.class)
+ public void testGetOututsTableNameNull() throws Exception {
+ hiveLineageService.getOutputs(null);
+ Assert.fail();
+ }
+
+ @Test (expectedExceptions = IllegalArgumentException.class)
+ public void testGetOutputsTableNameEmpty() throws Exception {
+ hiveLineageService.getOutputs("");
+ Assert.fail();
+ }
+
+ @Test (expectedExceptions = EntityNotFoundException.class)
+ public void testGetOutputsBadTableName() throws Exception {
+ hiveLineageService.getOutputs("blah");
+ Assert.fail();
+ }
+
+ @Test
+ public void testGetOutputsGraph() throws Exception {
+ JSONObject results = new JSONObject(hiveLineageService.getOutputsGraph("sales_fact"));
+ Assert.assertNotNull(results);
+ System.out.println("outputs graph = " + results);
+
+ JSONObject values = results.getJSONObject("values");
+ Assert.assertNotNull(values);
+
+ final JSONObject vertices = values.getJSONObject("vertices");
+ Assert.assertEquals(vertices.length(), 3);
+
+ final JSONObject edges = values.getJSONObject("edges");
+ Assert.assertEquals(edges.length(), 4);
+ }
+
+ @DataProvider(name = "tableNamesProvider")
+ private Object[][] tableNames() {
+ return new String[][] {
+ {"sales_fact", "4"},
+ {"time_dim", "3"},
+ {"sales_fact_daily_mv", "4"},
+ {"sales_fact_monthly_mv", "4"}
+ };
+ }
+
+ @Test (dataProvider = "tableNamesProvider")
+ public void testGetSchema(String tableName, String expected) throws Exception {
+ JSONObject results = new JSONObject(hiveLineageService.getSchema(tableName));
+ Assert.assertNotNull(results);
+ System.out.println("columns = " + results);
+
+ JSONArray rows = results.getJSONArray("rows");
+ Assert.assertEquals(rows.length(), Integer.parseInt(expected));
+
+ for (int index = 0; index < rows.length(); index++) {
+ final JSONObject row = rows.getJSONObject(index);
+ Assert.assertNotNull(row.getString("name"));
+ Assert.assertNotNull(row.getString("comment"));
+ Assert.assertNotNull(row.getString("dataType"));
+ Assert.assertEquals(row.getString("$typeName$"), "hive_column");
+ }
+ }
+
+ @Test (expectedExceptions = IllegalArgumentException.class)
+ public void testGetSchemaTableNameNull() throws Exception {
+ hiveLineageService.getSchema(null);
+ Assert.fail();
+ }
+
+ @Test (expectedExceptions = IllegalArgumentException.class)
+ public void testGetSchemaTableNameEmpty() throws Exception {
+ hiveLineageService.getSchema("");
+ Assert.fail();
+ }
+
+ @Test (expectedExceptions = EntityNotFoundException.class)
+ public void testGetSchemaBadTableName() throws Exception {
+ hiveLineageService.getSchema("blah");
+ Assert.fail();
+ }
+
+ private void setUpTypes() throws Exception {
+ TypesDef typesDef = createTypeDefinitions();
+ String typesAsJSON = TypesSerialization.toJson(typesDef);
+ metadataService.createType(typesAsJSON);
+ }
+
+ private static final String DATABASE_TYPE = "hive_db";
+ private static final String HIVE_TABLE_TYPE = "hive_table";
+ private static final String COLUMN_TYPE = "hive_column";
+ private static final String HIVE_PROCESS_TYPE = "hive_process";
+ private static final String STORAGE_DESC_TYPE = "StorageDesc";
+ private static final String VIEW_TYPE = "View";
+
+ private TypesDef createTypeDefinitions() {
+ HierarchicalTypeDefinition<ClassType> dbClsDef
+ = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("description", DataTypes.STRING_TYPE),
+ attrDef("locationUri", DataTypes.STRING_TYPE),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.LONG_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> storageDescClsDef =
+ TypesUtil.createClassTypeDef(STORAGE_DESC_TYPE, null,
+ attrDef("location", DataTypes.STRING_TYPE),
+ attrDef("inputFormat", DataTypes.STRING_TYPE),
+ attrDef("outputFormat", DataTypes.STRING_TYPE),
+ attrDef("compressed", DataTypes.STRING_TYPE,
+ Multiplicity.REQUIRED, false, null)
+ );
+
+ HierarchicalTypeDefinition<ClassType> columnClsDef =
+ TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("dataType", DataTypes.STRING_TYPE),
+ attrDef("comment", DataTypes.STRING_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> tblClsDef =
+ TypesUtil.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.LONG_TYPE),
+ attrDef("lastAccessTime", DataTypes.LONG_TYPE),
+ attrDef("tableType", DataTypes.STRING_TYPE),
+ attrDef("temporary", DataTypes.BOOLEAN_TYPE),
+ new AttributeDefinition("db", DATABASE_TYPE,
+ Multiplicity.REQUIRED, false, null),
+ // todo - uncomment this, something is broken
+// new AttributeDefinition("sd", STORAGE_DESC_TYPE,
+// Multiplicity.REQUIRED, true, null),
+ new AttributeDefinition("columns",
+ DataTypes.arrayTypeName(COLUMN_TYPE),
+ Multiplicity.COLLECTION, true, null)
+ );
+
+ HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
+ TypesUtil.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
+ attrDef("userName", DataTypes.STRING_TYPE),
+ attrDef("startTime", DataTypes.LONG_TYPE),
+ attrDef("endTime", DataTypes.LONG_TYPE),
+ attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
+ );
+
+ HierarchicalTypeDefinition<ClassType> viewClsDef =
+ TypesUtil.createClassTypeDef(VIEW_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("db", DATABASE_TYPE,
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("inputTables",
+ DataTypes.arrayTypeName(HIVE_TABLE_TYPE),
+ Multiplicity.COLLECTION, false, null)
+ );
+
+ HierarchicalTypeDefinition<TraitType> dimTraitDef =
+ TypesUtil.createTraitTypeDef("Dimension", null);
+
+ HierarchicalTypeDefinition<TraitType> factTraitDef =
+ TypesUtil.createTraitTypeDef("Fact", null);
+
+ HierarchicalTypeDefinition<TraitType> metricTraitDef =
+ TypesUtil.createTraitTypeDef("Metric", null);
+
+ HierarchicalTypeDefinition<TraitType> etlTraitDef =
+ TypesUtil.createTraitTypeDef("ETL", null);
+
+ HierarchicalTypeDefinition<TraitType> piiTraitDef =
+ TypesUtil.createTraitTypeDef("PII", null);
+
+ HierarchicalTypeDefinition<TraitType> jdbcTraitDef =
+ TypesUtil.createTraitTypeDef("JdbcAccess", null);
+
+ return TypeUtils.getTypesDef(
+ ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(dimTraitDef, factTraitDef,
+ piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
+ ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef,
+ tblClsDef, loadProcessClsDef, viewClsDef)
+ );
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT) {
+ return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
+ return attrDef(name, dT, m, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT,
+ Multiplicity m, boolean isComposite, String reverseAttributeName) {
+ Preconditions.checkNotNull(name);
+ Preconditions.checkNotNull(dT);
+ return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
+ }
+
+ private void setupInstances() throws Exception {
+ Id salesDB = database(
+ "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
+
+ Referenceable sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales",
+ "TextInputFormat", "TextOutputFormat", true);
+
+ List<Referenceable> salesFactColumns = ImmutableList.of(
+ column("time_id", "int", "time id"),
+ column("product_id", "int", "product id"),
+ column("customer_id", "int", "customer id", "PII"),
+ column("sales", "double", "product id", "Metric")
+ );
+
+ Id salesFact = table("sales_fact", "sales fact table",
+ salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
+
+ List<Referenceable> timeDimColumns = ImmutableList.of(
+ column("time_id", "int", "time id"),
+ column("dayOfYear", "int", "day Of Year"),
+ column("weekDay", "int", "week Day")
+ );
+
+ Id timeDim = table("time_dim", "time dimension table",
+ salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
+
+ Id reportingDB = database("Reporting", "reporting database", "Jane BI",
+ "hdfs://host:8000/apps/warehouse/reporting");
+
+ Id salesFactDaily = table("sales_fact_daily_mv",
+ "sales fact daily materialized view",
+ reportingDB, sd, "Joe BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL",
+ ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+
+ List<Referenceable> productDimColumns = ImmutableList.of(
+ column("product_id", "int", "product id"),
+ column("product_name", "string", "product name"),
+ column("brand_name", "int", "brand name")
+ );
+
+ Id productDim = table("product_dim", "product dimension table",
+ salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
+
+ view("product_dim_view", reportingDB,
+ ImmutableList.of(productDim), "Dimension", "JdbcAccess");
+
+ List<Referenceable> customerDimColumns = ImmutableList.of(
+ column("customer_id", "int", "customer id", "PII"),
+ column("name", "string", "customer name", "PII"),
+ column("address", "string", "customer address", "PII")
+ );
+
+ Id customerDim = table("customer_dim", "customer dimension table",
+ salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
+
+ view("customer_dim_view", reportingDB,
+ ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
+
+ Id salesFactMonthly = table("sales_fact_monthly_mv",
+ "sales fact monthly materialized view",
+ reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL",
+ ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+ }
+
+ Id database(String name, String description,
+ String owner, String locationUri,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("locationUri", locationUri);
+ referenceable.set("createTime", System.currentTimeMillis());
+
+ return createInstance(referenceable);
+ }
+
+ Referenceable storageDescriptor(String location, String inputFormat,
+ String outputFormat,
+ boolean compressed) throws Exception {
+ Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
+ referenceable.set("location", location);
+ referenceable.set("inputFormat", inputFormat);
+ referenceable.set("outputFormat", outputFormat);
+ referenceable.set("compressed", compressed);
+
+ return referenceable;
+ }
+
+ Referenceable column(String name, String dataType, String comment,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("dataType", dataType);
+ referenceable.set("comment", comment);
+
+ return referenceable;
+ }
+
+ Id table(String name, String description,
+ Id dbId, Referenceable sd,
+ String owner, String tableType,
+ List<Referenceable> columns,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("tableType", tableType);
+ referenceable.set("temporary", false);
+ referenceable.set("createTime", System.currentTimeMillis());
+ referenceable.set("lastAccessTime", System.currentTimeMillis());
+ referenceable.set("retention", System.currentTimeMillis());
+
+ referenceable.set("db", dbId);
+ // todo - uncomment this, something is broken
+ // referenceable.set("sd", sd);
+ referenceable.set("columns", columns);
+
+ return createInstance(referenceable);
+ }
+
+ Id loadProcess(String name, String description, String user,
+ List<Id> inputTables,
+ List<Id> outputTables,
+ String queryText, String queryPlan,
+ String queryId, String queryGraph,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("user", user);
+ referenceable.set("startTime", System.currentTimeMillis());
+ referenceable.set("endTime", System.currentTimeMillis() + 10000);
+
+ referenceable.set("inputs", inputTables);
+ referenceable.set("outputs", outputTables);
+
+ referenceable.set("queryText", queryText);
+ referenceable.set("queryPlan", queryPlan);
+ referenceable.set("queryId", queryId);
+ referenceable.set("queryGraph", queryGraph);
+
+ return createInstance(referenceable);
+ }
+
+ Id view(String name, Id dbId,
+ List<Id> inputTables,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("db", dbId);
+
+ referenceable.set("inputTables", inputTables);
+
+ return createInstance(referenceable);
+ }
+
+ private Id createInstance(Referenceable referenceable) throws Exception {
+ String typeName = referenceable.getTypeName();
+ System.out.println("creating instance of type " + typeName);
+
+ String entityJSON = InstanceSerialization.toJson(referenceable, true);
+ System.out.println("Submitting new entity= " + entityJSON);
+ String guid = metadataService.createEntity(entityJSON);
+ System.out.println("created instance for type " + typeName + ", guid: " + guid);
+
+ // return the reference to created instance with guid
+ return new Id(guid, 0, referenceable.getTypeName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/test/java/org/apache/atlas/repository/BaseTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/BaseTest.java b/repository/src/test/java/org/apache/atlas/repository/BaseTest.java
new file mode 100755
index 0000000..1b1f692
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/BaseTest.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.atlas.MetadataException;
+import org.apache.atlas.repository.memory.MemRepository;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.junit.Before;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.Map;
+
+public abstract class BaseTest {
+
+ public static final String STRUCT_TYPE_1 = "t1";
+ public static final String STRUCT_TYPE_2 = "t2";
+ public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
+ public static final long TEST_DATE_IN_LONG=1418265358440L;
+ protected IRepository repo;
+
+ public static Struct createStruct() throws MetadataException {
+ StructType structType = (StructType) TypeSystem.getInstance()
+ .getDataType(StructType.class, STRUCT_TYPE_1);
+ Struct s = new Struct(structType.getName());
+ s.set("a", 1);
+ s.set("b", true);
+ s.set("c", (byte) 1);
+ s.set("d", (short) 2);
+ s.set("e", 1);
+ s.set("f", 1);
+ s.set("g", 1L);
+ s.set("h", 1.0f);
+ s.set("i", 1.0);
+ s.set("j", BigInteger.valueOf(1L));
+ s.set("k", new BigDecimal(1));
+ s.set("l", new Date(1418265358440L));
+ s.set("m", Lists.<Integer>asList(Integer.valueOf(1), new Integer[]{Integer.valueOf(1)}));
+ s.set("n", Lists.<BigDecimal>asList(BigDecimal.valueOf(1.1),
+ new BigDecimal[]{BigDecimal.valueOf(1.1)}));
+ Map<String, Double> hm = Maps.<String, Double>newHashMap();
+ hm.put("a", 1.0);
+ hm.put("b", 2.0);
+ s.set("o", hm);
+ return s;
+ }
+
+ protected final TypeSystem getTypeSystem() {
+ return TypeSystem.getInstance();
+ }
+
+ protected final IRepository getRepository() {
+ return repo;
+ }
+
+ @Before
+ public void setup() throws Exception {
+
+ TypeSystem ts = TypeSystem.getInstance();
+ ts.reset();
+ repo = new MemRepository(ts);
+
+ StructType structType = ts.defineStructType(STRUCT_TYPE_1,
+ true,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
+ TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
+ TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
+ TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
+ TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
+ TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
+ TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
+ TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
+ TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
+ TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ TypesUtil.createOptionalAttrDef("o",
+ ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
+
+ StructType recursiveStructType = ts.defineStructType(STRUCT_TYPE_2,
+ true,
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
+ TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
+ }
+
+ protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs)
+ throws MetadataException {
+
+ return getTypeSystem().defineTraitTypes(tDefs);
+ }
+
+ /*
+ * Class Hierarchy is:
+ * Department(name : String, employees : Array[Person])
+ * Person(name : String, department : Department, manager : Manager)
+ * Manager(subordinates : Array[Person]) extends Person
+ *
+ * Persons can have SecurityClearance(level : Int) clearance.
+ */
+ protected void defineDeptEmployeeTypes(TypeSystem ts) throws MetadataException {
+
+ HierarchicalTypeDefinition<ClassType> deptTypeDef =
+ TypesUtil.createClassTypeDef("Department", ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, true, "department")
+ );
+ HierarchicalTypeDefinition<ClassType> personTypeDef =
+ TypesUtil.createClassTypeDef("Person", ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department",
+ "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager",
+ "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ );
+ HierarchicalTypeDefinition<ClassType> managerTypeDef =
+ TypesUtil.createClassTypeDef("Manager",
+ ImmutableList.<String>of("Person"),
+ new AttributeDefinition("subordinates",
+ String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, false, "manager")
+ );
+
+ HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef =
+ TypesUtil.createTraitTypeDef("SecurityClearance",
+ ImmutableList.<String>of(),
+ TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE)
+ );
+
+ ts.defineTypes(ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(securityClearanceTypeDef),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of(deptTypeDef, personTypeDef,
+ managerTypeDef));
+
+ ImmutableList<HierarchicalType> types = ImmutableList.of(
+ ts.getDataType(HierarchicalType.class, "SecurityClearance"),
+ ts.getDataType(ClassType.class, "Department"),
+ ts.getDataType(ClassType.class, "Person"),
+ ts.getDataType(ClassType.class, "Manager")
+ );
+
+ repo.defineTypes(types);
+
+ }
+
+ protected Referenceable createDeptEg1(TypeSystem ts) throws MetadataException {
+ Referenceable hrDept = new Referenceable("Department");
+ Referenceable john = new Referenceable("Person");
+ Referenceable jane = new Referenceable("Manager", "SecurityClearance");
+
+ hrDept.set("name", "hr");
+ john.set("name", "John");
+ john.set("department", hrDept);
+ jane.set("name", "Jane");
+ jane.set("department", hrDept);
+
+ john.set("manager", jane);
+
+ hrDept.set("employees", ImmutableList.<Referenceable>of(john, jane));
+
+ jane.set("subordinates", ImmutableList.<Referenceable>of(john));
+
+ jane.getTrait("SecurityClearance").set("level", 1);
+
+ ClassType deptType = ts.getDataType(ClassType.class, "Department");
+ ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
+
+ return hrDept;
+ }
+}
[19/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
Posted by ve...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/atlas/tools/hive/HiveMockMetadataService.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/atlas/tools/hive/HiveMockMetadataService.scala b/tools/src/test/scala/org/apache/atlas/tools/hive/HiveMockMetadataService.scala
new file mode 100755
index 0000000..6c90082
--- /dev/null
+++ b/tools/src/test/scala/org/apache/atlas/tools/hive/HiveMockMetadataService.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.hive
+
+object HiveMockMetadataService {
+
+ def getTable(dbName: String, table: String): Table = {
+ return Table(dbName, table,
+ StorageDescriptor(
+ List[FieldSchema](
+ FieldSchema("d_date_sk", "int", null),
+ FieldSchema("d_date_id", "string", null),
+ FieldSchema("d_date", "string", null),
+ FieldSchema("d_month_seq", "int", null),
+ FieldSchema("d_week_seq", "int", null),
+ FieldSchema("d_quarter_seq", "int", null),
+ FieldSchema("d_year", "int", null),
+ FieldSchema("d_dow", "int", null),
+ FieldSchema("d_moy", "int", null),
+ FieldSchema("d_dom", "int", null),
+ FieldSchema("d_qoy", "int", null),
+ FieldSchema("d_fy_year", "int", null),
+ FieldSchema("d_fy_quarter_seq", "int", null),
+ FieldSchema("d_fy_week_seq", "int", null),
+ FieldSchema("d_day_name", "string", null),
+ FieldSchema("d_quarter_name", "string", null),
+ FieldSchema("d_holiday", "string", null),
+ FieldSchema("d_weekend", "string", null),
+ FieldSchema("d_following_holiday", "string", null),
+ FieldSchema("d_first_dom", "int", null),
+ FieldSchema("d_last_dom", "int", null),
+ FieldSchema("d_same_day_ly", "int", null),
+ FieldSchema("d_same_day_lq", "int", null),
+ FieldSchema("d_current_day", "string", null),
+ FieldSchema("d_current_week", "string", null),
+ FieldSchema("d_current_month", "string", null),
+ FieldSchema("d_current_quarter", "string", null),
+ FieldSchema("d_current_year", "string", null)
+ ),
+ "file:/tmp/warehouse/tpcds.db/date_dim",
+ "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+ "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+ false,
+ 0, List[String](), List[String](),
+ Map[String, String](),
+ false
+ ),
+ Map[String, String](),
+ "Table")
+ }
+
+ case class FieldSchema(name: String, typeName: String, comment: String)
+
+ case class SerDe(name: String, serializationLib: String, parameters: Map[String, String])
+
+ case class StorageDescriptor(fields: List[FieldSchema],
+ location: String, inputFormat: String,
+ outputFormat: String, compressed: Boolean,
+ numBuckets: Int, bucketColumns: List[String],
+ sortColumns: List[String],
+ parameters: Map[String, String],
+ storedAsSubDirs: Boolean
+ )
+
+ case class Table(dbName: String, tableName: String, storageDesc: StorageDescriptor,
+ parameters: Map[String, String],
+ tableType: String)
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftLexerTest.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftLexerTest.scala b/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftLexerTest.scala
new file mode 100755
index 0000000..000b883
--- /dev/null
+++ b/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftLexerTest.scala
@@ -0,0 +1,525 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.thrift
+
+import org.junit.{Assert, Test}
+
+import scala.util.parsing.input.CharArrayReader
+
+class ThriftLexerTest {
+
+ @Test def testSimple {
+ val p = new ThriftParser
+ val r = scan(p, """efg abc""")
+ Assert.assertTrue(r.successful)
+
+ }
+
+ @Test def testStruct {
+ val p = new ThriftParser
+ val r = scan(p, """struct PartitionSpecWithSharedSD {
+ 1: list<PartitionWithoutSD> partitions,
+ 2: StorageDescriptor sd,
+ }""")
+ Assert.assertTrue(r.successful)
+
+ }
+
+ @Test def testTableStruct {
+ val p = new ThriftParser
+ val r = scan(p, """// table information
+ struct Table {
+ 1: string tableName, // name of the table
+ 2: string dbName, // database name ('default')
+ 3: string owner, // owner of this table
+ 4: i32 createTime, // creation time of the table
+ 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
+ 6: i32 retention, // retention time
+ 7: StorageDescriptor sd, // storage descriptor of the table
+ 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
+ 9: map<string, string> parameters, // to store comments or any other user level parameters
+ 10: string viewOriginalText, // original view text, null for non-view
+ 11: string viewExpandedText, // expanded view text, null for non-view
+ 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
+ 13: optional PrincipalPrivilegeSet privileges,
+ 14: optional bool temporary=false
+ }""")
+ Assert.assertTrue(r.successful)
+
+ }
+
+ @Test def testIncorrectStruct {
+ val p = new ThriftParser
+ val r = scan(p, """// table information
+ struct Table {
+ | 1: string tableName, // name of the table
+ | 2: string dbName
+ }""")
+ Assert.assertFalse(r.successful)
+
+ }
+
+ @Test def testNegativeInt {
+ val p = new ThriftParser
+ val r = scan(p, """-1""")
+ Assert.assertTrue(r.successful)
+
+ }
+
+ @Test def testComment {
+ val p = new ThriftParser
+ val r = scan(p, """/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */""")
+ }
+
+ def scan(p: ThriftParser, str: String): p.lexical.ParseResult[_] = {
+ val l = p.lexical
+ var s: l.Input = new CharArrayReader(str.toCharArray)
+ var r = (l.whitespace.? ~ l.token)(s)
+ s = r.next
+
+ while (r.successful && !s.atEnd) {
+ s = r.next
+ if (!s.atEnd) {
+ r = (l.whitespace.? ~ l.token)(s)
+ }
+ }
+ r.asInstanceOf[p.lexical.ParseResult[_]]
+ }
+
+ @Test def testService {
+ val p = new ThriftParser
+ val r = scan(p, """/**
+ * This interface is live.
+ */
+ service ThriftHiveMetastore extends fb303.FacebookService
+ {
+ string getMetaConf(1:string key) throws(1:MetaException o1)
+ void setMetaConf(1:string key, 2:string value) throws(1:MetaException o1)
+
+ void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
+ Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ void drop_database(1:string name, 2:bool deleteData, 3:bool cascade) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
+ list<string> get_databases(1:string pattern) throws(1:MetaException o1)
+ list<string> get_all_databases() throws(1:MetaException o1)
+ void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // returns the type with given name (make seperate calls for the dependent types if needed)
+ Type get_type(1:string name) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ bool create_type(1:Type type) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
+ bool drop_type(1:string type) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ map<string, Type> get_type_all(1:string name)
+ throws(1:MetaException o2)
+
+ // Gets a list of FieldSchemas describing the columns of a particular table
+ list<FieldSchema> get_fields(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3),
+
+ // Gets a list of FieldSchemas describing both the columns and the partition keys of a particular table
+ list<FieldSchema> get_schema(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
+
+ // create a Hive table. Following fields must be set
+ // tableName
+ // database (only 'default' for now until Hive QL supports databases)
+ // owner (not needed, but good to have for tracking purposes)
+ // sd.cols (list of field schemas)
+ // sd.inputFormat (SequenceFileInputFormat (binary like falcon tables or u_full) or TextInputFormat)
+ // sd.outputFormat (SequenceFileInputFormat (binary) or TextInputFormat)
+ // sd.serdeInfo.serializationLib (SerDe class name eg org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe
+ // * See notes on DDL_TIME
+ void create_table(1:Table tbl) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4)
+ void create_table_with_environment_context(1:Table tbl,
+ 2:EnvironmentContext environment_context)
+ throws (1:AlreadyExistsException o1,
+ 2:InvalidObjectException o2, 3:MetaException o3,
+ 4:NoSuchObjectException o4)
+ // drops the table and all the partitions associated with it if the table has partitions
+ // delete data (including partitions) if deleteData is set to true
+ void drop_table(1:string dbname, 2:string name, 3:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o3)
+ void drop_table_with_environment_context(1:string dbname, 2:string name, 3:bool deleteData,
+ 4:EnvironmentContext environment_context)
+ throws(1:NoSuchObjectException o1, 2:MetaException o3)
+ list<string> get_tables(1: string db_name, 2: string pattern) throws (1: MetaException o1)
+ list<string> get_all_tables(1: string db_name) throws (1: MetaException o1)
+
+ Table get_table(1:string dbname, 2:string tbl_name)
+ throws (1:MetaException o1, 2:NoSuchObjectException o2)
+ list<Table> get_table_objects_by_name(1:string dbname, 2:list<string> tbl_names)
+ throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
+
+ // Get a list of table names that match a filter.
+ // The filter operators are LIKE, <, <=, >, >=, =, <>
+ //
+ // In the filter statement, values interpreted as strings must be enclosed in quotes,
+ // while values interpreted as integers should not be. Strings and integers are the only
+ // supported value types.
+ //
+ // The currently supported key names in the filter are:
+ // Constants.HIVE_FILTER_FIELD_OWNER, which filters on the tables' owner's name
+ // and supports all filter operators
+ // Constants.HIVE_FILTER_FIELD_LAST_ACCESS, which filters on the last access times
+ // and supports all filter operators except LIKE
+ // Constants.HIVE_FILTER_FIELD_PARAMS, which filters on the tables' parameter keys and values
+ // and only supports the filter operators = and <>.
+ // Append the parameter key name to HIVE_FILTER_FIELD_PARAMS in the filter statement.
+ // For example, to filter on parameter keys called "retention", the key name in the filter
+ // statement should be Constants.HIVE_FILTER_FIELD_PARAMS + "retention"
+ // Also, = and <> only work for keys that exist
+ // in the tables. E.g., if you are looking for tables where key1 <> value, it will only
+ // look at tables that have a value for the parameter key1.
+ // Some example filter statements include:
+ // filter = Constants.HIVE_FILTER_FIELD_OWNER + " like \".*test.*\" and " +
+ // Constants.HIVE_FILTER_FIELD_LAST_ACCESS + " = 0";
+ // filter = Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"30\" or " +
+ // Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"90\""
+ // @param dbName
+ // The name of the database from which you will retrieve the table names
+ // @param filterType
+ // The type of filter
+ // @param filter
+ // The filter string
+ // @param max_tables
+ // The maximum number of tables returned
+ // @return A list of table names that match the desired filter
+ list<string> get_table_names_by_filter(1:string dbname, 2:string filter, 3:i16 max_tables=-1)
+ throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
+
+ // alter table applies to only future partitions not for existing partitions
+ // * See notes on DDL_TIME
+ void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ void alter_table_with_environment_context(1:string dbname, 2:string tbl_name,
+ 3:Table new_tbl, 4:EnvironmentContext environment_context)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ // the following applies to only tables that have partitions
+ // * See notes on DDL_TIME
+ Partition add_partition(1:Partition new_part)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition add_partition_with_environment_context(1:Partition new_part,
+ 2:EnvironmentContext environment_context)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2,
+ 3:MetaException o3)
+ i32 add_partitions(1:list<Partition> new_parts)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ i32 add_partitions_pspec(1:list<PartitionSpec> new_parts)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ AddPartitionsResult add_partitions_req(1:AddPartitionsRequest request)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:list<string> part_vals, 4:EnvironmentContext environment_context)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ Partition append_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:string part_name, 4:EnvironmentContext environment_context)
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ bool drop_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ bool drop_partition_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:list<string> part_vals, 4:bool deleteData, 5:EnvironmentContext environment_context)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ bool drop_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name, 4:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ bool drop_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
+ 3:string part_name, 4:bool deleteData, 5:EnvironmentContext environment_context)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ DropPartitionsResult drop_partitions_req(1: DropPartitionsRequest req)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ Partition get_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ Partition exchange_partition(1:map<string, string> partitionSpecs, 2:string source_db,
+ 3:string source_table_name, 4:string dest_db, 5:string dest_table_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
+
+ Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals,
+ 4: string user_name, 5: list<string> group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ Partition get_partition_by_name(1:string db_name 2:string tbl_name, 3:string part_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // returns all the partitions for this table in reverse chronological order.
+ // If max parts is given then it will return only that many.
+ list<Partition> get_partitions(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1,
+ 4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ list<PartitionSpec> get_partitions_pspec(1:string db_name, 2:string tbl_name, 3:i32 max_parts=-1)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
+ throws(1:MetaException o2)
+
+ // get_partition*_ps methods allow filtering by a partial partition specification,
+ // as needed for dynamic partitions. The values that are not restricted should
+ // be empty strings. Nulls were considered (instead of "") but caused errors in
+ // generated Python code. The size of part_vals may be smaller than the
+ // number of partition columns - the unspecified values are considered the same
+ // as "".
+ list<Partition> get_partitions_ps(1:string db_name 2:string tbl_name
+ 3:list<string> part_vals, 4:i16 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ list<Partition> get_partitions_ps_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1,
+ 5: string user_name, 6: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
+ list<string> get_partition_names_ps(1:string db_name,
+ 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // get the partitions matching the given partition filter
+ list<Partition> get_partitions_by_filter(1:string db_name 2:string tbl_name
+ 3:string filter, 4:i16 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // List partitions as PartitionSpec instances.
+ list<PartitionSpec> get_part_specs_by_filter(1:string db_name 2:string tbl_name
+ 3:string filter, 4:i32 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // get the partitions matching the given partition filter
+ // unlike get_partitions_by_filter, takes serialized hive expression, and with that can work
+ // with any filter (get_partitions_by_filter only works if the filter can be pushed down to JDOQL.
+ PartitionsByExprResult get_partitions_by_expr(1:PartitionsByExprRequest req)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // get partitions give a list of partition names
+ list<Partition> get_partitions_by_names(1:string db_name 2:string tbl_name 3:list<string> names)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ // changes the partition to the new partition object. partition is identified from the part values
+ // in the new_part
+ // * See notes on DDL_TIME
+ void alter_partition(1:string db_name, 2:string tbl_name, 3:Partition new_part)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ // change a list of partitions. All partitions are altered atomically and all
+ // prehooks are fired together followed by all post hooks
+ void alter_partitions(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ void alter_partition_with_environment_context(1:string db_name,
+ 2:string tbl_name, 3:Partition new_part,
+ 4:EnvironmentContext environment_context)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ // rename the old partition to the new partition object by changing old part values to the part values
+ // in the new_part. old partition is identified from part_vals.
+ // partition keys in new_part should be the same as those in old partition.
+ void rename_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:Partition new_part)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ // returns whether or not the partition name is valid based on the value of the config
+ // hive.metastore.partition.name.whitelist.pattern
+ bool partition_name_has_valid_characters(1:list<string> part_vals, 2:bool throw_exception)
+ throws(1: MetaException o1)
+
+ // gets the value of the configuration key in the metastore server. returns
+ // defaultValue if the key does not exist. if the configuration key does not
+ // begin with "hive", "mapred", or "hdfs", a ConfigValSecurityException is
+ // thrown.
+ string get_config_value(1:string name, 2:string defaultValue)
+ throws(1:ConfigValSecurityException o1)
+
+ // converts a partition name into a partition values array
+ list<string> partition_name_to_vals(1: string part_name)
+ throws(1: MetaException o1)
+ // converts a partition name into a partition specification (a mapping from
+ // the partition cols to the values)
+ map<string, string> partition_name_to_spec(1: string part_name)
+ throws(1: MetaException o1)
+
+ void markPartitionForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
+ 4:PartitionEventType eventType) throws (1: MetaException o1, 2: NoSuchObjectException o2,
+ 3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
+ 6: InvalidPartitionException o6)
+ bool isPartitionMarkedForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
+ 4: PartitionEventType eventType) throws (1: MetaException o1, 2:NoSuchObjectException o2,
+ 3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
+ 6: InvalidPartitionException o6)
+
+ //index
+ Index add_index(1:Index new_index, 2: Table index_table)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ void alter_index(1:string dbname, 2:string base_tbl_name, 3:string idx_name, 4:Index new_idx)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+ bool drop_index_by_name(1:string db_name, 2:string tbl_name, 3:string index_name, 4:bool deleteData)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ Index get_index_by_name(1:string db_name 2:string tbl_name, 3:string index_name)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ list<Index> get_indexes(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ list<string> get_index_names(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
+ throws(1:MetaException o2)
+
+ // column statistics interfaces
+
+ // update APIs persist the column statistics object(s) that are passed in. If statistics already
+ // exists for one or more columns, the existing statistics will be overwritten. The update APIs
+ // validate that the dbName, tableName, partName, colName[] passed in as part of the ColumnStatistics
+ // struct are valid, throws InvalidInputException/NoSuchObjectException if found to be invalid
+ bool update_table_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
+ 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+ bool update_partition_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
+ 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+
+ // get APIs return the column statistics corresponding to db_name, tbl_name, [part_name], col_name if
+ // such statistics exists. If the required statistics doesn't exist, get APIs throw NoSuchObjectException
+ // For instance, if get_table_column_statistics is called on a partitioned table for which only
+ // partition level column stats exist, get_table_column_statistics will throw NoSuchObjectException
+ ColumnStatistics get_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidInputException o3, 4:InvalidObjectException o4)
+ ColumnStatistics get_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name,
+ 4:string col_name) throws (1:NoSuchObjectException o1, 2:MetaException o2,
+ 3:InvalidInputException o3, 4:InvalidObjectException o4)
+ TableStatsResult get_table_statistics_req(1:TableStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2)
+ PartitionsStatsResult get_partitions_statistics_req(1:PartitionsStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2)
+ AggrStats get_aggr_stats_for(1:PartitionsStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2)
+ bool set_aggr_stats_for(1:SetPartitionsStatsRequest request) throws
+ (1:NoSuchObjectException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+
+
+ // delete APIs attempt to delete column statistics, if found, associated with a given db_name, tbl_name, [part_name]
+ // and col_name. If the delete API doesn't find the statistics record in the metastore, throws NoSuchObjectException
+ // Delete API validates the input and if the input is invalid throws InvalidInputException/InvalidObjectException.
+ bool delete_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name, 4:string col_name) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
+ bool delete_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
+ (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
+ 4:InvalidInputException o4)
+
+ //
+ // user-defined functions
+ //
+
+ void create_function(1:Function func)
+ throws (1:AlreadyExistsException o1,
+ 2:InvalidObjectException o2,
+ 3:MetaException o3,
+ 4:NoSuchObjectException o4)
+
+ void drop_function(1:string dbName, 2:string funcName)
+ throws (1:NoSuchObjectException o1, 2:MetaException o3)
+
+ void alter_function(1:string dbName, 2:string funcName, 3:Function newFunc)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ list<string> get_functions(1:string dbName, 2:string pattern)
+ throws (1:MetaException o1)
+ Function get_function(1:string dbName, 2:string funcName)
+ throws (1:MetaException o1, 2:NoSuchObjectException o2)
+
+ //authorization privileges
+
+ bool create_role(1:Role role) throws(1:MetaException o1)
+ bool drop_role(1:string role_name) throws(1:MetaException o1)
+ list<string> get_role_names() throws(1:MetaException o1)
+ // Deprecated, use grant_revoke_role()
+ bool grant_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type,
+ 4:string grantor, 5:PrincipalType grantorType, 6:bool grant_option) throws(1:MetaException o1)
+ // Deprecated, use grant_revoke_role()
+ bool revoke_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type)
+ throws(1:MetaException o1)
+ list<Role> list_roles(1:string principal_name, 2:PrincipalType principal_type) throws(1:MetaException o1)
+ GrantRevokeRoleResponse grant_revoke_role(1:GrantRevokeRoleRequest request) throws(1:MetaException o1)
+
+ // get all role-grants for users/roles that have been granted the given role
+ // Note that in the returned list of RolePrincipalGrants, the roleName is
+ // redundant as it would match the role_name argument of this function
+ GetPrincipalsInRoleResponse get_principals_in_role(1: GetPrincipalsInRoleRequest request) throws(1:MetaException o1)
+
+ // get grant information of all roles granted to the given principal
+ // Note that in the returned list of RolePrincipalGrants, the principal name,type is
+ // redundant as it would match the principal name,type arguments of this function
+ GetRoleGrantsForPrincipalResponse get_role_grants_for_principal(1: GetRoleGrantsForPrincipalRequest request) throws(1:MetaException o1)
+
+ PrincipalPrivilegeSet get_privilege_set(1:HiveObjectRef hiveObject, 2:string user_name,
+ 3: list<string> group_names) throws(1:MetaException o1)
+ list<HiveObjectPrivilege> list_privileges(1:string principal_name, 2:PrincipalType principal_type,
+ 3: HiveObjectRef hiveObject) throws(1:MetaException o1)
+
+ // Deprecated, use grant_revoke_privileges()
+ bool grant_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
+ // Deprecated, use grant_revoke_privileges()
+ bool revoke_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
+ GrantRevokePrivilegeResponse grant_revoke_privileges(1:GrantRevokePrivilegeRequest request) throws(1:MetaException o1);
+
+ // this is used by metastore client to send UGI information to metastore server immediately
+ // after setting up a connection.
+ list<string> set_ugi(1:string user_name, 2:list<string> group_names) throws (1:MetaException o1)
+
+ //Authentication (delegation token) interfaces
+
+ // get metastore server delegation token for use from the map/reduce tasks to authenticate
+ // to metastore server
+ string get_delegation_token(1:string token_owner, 2:string renewer_kerberos_principal_name)
+ throws (1:MetaException o1)
+
+ // method to renew delegation token obtained from metastore server
+ i64 renew_delegation_token(1:string token_str_form) throws (1:MetaException o1)
+
+ // method to cancel delegation token obtained from metastore server
+ void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1)
+
+ // Transaction and lock management calls
+ // Get just list of open transactions
+ GetOpenTxnsResponse get_open_txns()
+ // Get list of open transactions with state (open, aborted)
+ GetOpenTxnsInfoResponse get_open_txns_info()
+ OpenTxnsResponse open_txns(1:OpenTxnRequest rqst)
+ void abort_txn(1:AbortTxnRequest rqst) throws (1:NoSuchTxnException o1)
+ void commit_txn(1:CommitTxnRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+ LockResponse lock(1:LockRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+ LockResponse check_lock(1:CheckLockRequest rqst)
+ throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2, 3:NoSuchLockException o3)
+ void unlock(1:UnlockRequest rqst) throws (1:NoSuchLockException o1, 2:TxnOpenException o2)
+ ShowLocksResponse show_locks(1:ShowLocksRequest rqst)
+ void heartbeat(1:HeartbeatRequest ids) throws (1:NoSuchLockException o1, 2:NoSuchTxnException o2, 3:TxnAbortedException o3)
+ HeartbeatTxnRangeResponse heartbeat_txn_range(1:HeartbeatTxnRangeRequest txns)
+ void compact(1:CompactionRequest rqst)
+ ShowCompactResponse show_compact(1:ShowCompactRequest rqst)
+ }""")
+ Assert.assertTrue(r.successful)
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftParserTest.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftParserTest.scala b/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftParserTest.scala
new file mode 100755
index 0000000..82e5383
--- /dev/null
+++ b/tools/src/test/scala/org/apache/atlas/tools/thrift/ThriftParserTest.scala
@@ -0,0 +1,772 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.tools.thrift
+
+import com.google.gson.JsonParser
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.{write => swrite}
+import org.json4s.{NoTypeHints, _}
+import org.junit.{Assert, Test}
+
+import scala.io.Source
+import scala.reflect.ClassTag
+
+/**
+ * Copied from
+ * [[https://github.com/json4s/json4s/blob/master/ext/src/main/scala/org/json4s/ext/EnumSerializer.scala json4s github]]
+ * to avoid dependency on json4s-ext.
+ */
+class EnumNameSerializer[E <: Enumeration: ClassTag](enum: E) extends Serializer[E#Value] {
+ import JsonDSL._
+
+ val EnumerationClass = classOf[E#Value]
+
+ def deserialize(implicit format: Formats): PartialFunction[(TypeInfo, JValue), E#Value] = {
+ case (t@TypeInfo(EnumerationClass, _), json) if (isValid(json)) => {
+ json match {
+ case JString(value) => enum.withName(value)
+ case value => throw new MappingException("Can't convert " +
+ value + " to " + EnumerationClass)
+ }
+ }
+ }
+
+ private[this] def isValid(json: JValue) = json match {
+ case JString(value) if (enum.values.exists(_.toString == value)) => true
+ case _ => false
+ }
+
+ def serialize(implicit format: Formats): PartialFunction[Any, JValue] = {
+ case i: E#Value => i.toString
+ }
+}
+
+class ThriftParserTest {
+
+ @Test def testSimple {
+ var p = new ThriftParser
+ val parser = new JsonParser
+
+ var td: Option[ThriftDef] = p( """include "share/fb303/if/fb303.thrift"
+
+ namespace java org.apache.hadoop.hive.metastore.api
+ namespace php metastore
+ namespace cpp Apache.Hadoop.Hive
+ """)
+
+ val parsed = parser.parse(toJson(td.get))
+ val sample = parser.parse( """{
+ "includes":[{
+ "value":"share/fb303/if/fb303.thrift"
+ }],
+ "cppIncludes":[],
+ "namespaces":[{
+ "lang":"",
+ "name":"Apache.Hadoop.Hive",
+ "otherLang":"cpp"
+ },{
+ "lang":"",
+ "name":"metastore",
+ "otherLang":"php"
+ },{
+ "lang":"",
+ "name":"org.apache.hadoop.hive.metastore.api",
+ "otherLang":"java"
+ }],
+ "constants":[],
+ "typedefs":[],
+ "enums":[],
+ "senums":[],
+ "structs":[],
+ "unions":[],
+ "xceptions":[],
+ "services":[]
+}""")
+
+ Assert.assertEquals(parsed.toString, sample.toString)
+ }
+
+ @Test def testStruct {
+ val p = new ThriftParser
+ val parser = new JsonParser
+
+ var td: Option[ThriftDef] = p( """struct PartitionSpecWithSharedSD {
+ 1: list<PartitionWithoutSD> partitions,
+ 2: StorageDescriptor sd
+ }""")
+
+ val parsed = parser.parse(toJson(td.get))
+
+ val sample = parser.parse( """{
+ "includes":[],
+ "cppIncludes":[],
+ "namespaces":[],
+ "constants":[],
+ "typedefs":[],
+ "enums":[],
+ "senums":[],
+ "structs":[{
+ "name":"PartitionSpecWithSharedSD",
+ "xsdAll":false,
+ "fields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "elemType":{
+ "name":"PartitionWithoutSD"
+ }
+ },
+ "name":"partitions",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"StorageDescriptor"
+ },
+ "name":"sd",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ }],
+ "unions":[],
+ "xceptions":[],
+ "services":[]
+}""")
+
+ Assert.assertEquals(parsed.toString, sample.toString)
+ }
+
+ def toJson(td: ThriftDef) = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new EnumNameSerializer(BASE_TYPES) +
+ new EnumNameSerializer(THRIFT_LANG)
+ val ser = swrite(td)
+ pretty(render(parse(ser)))
+ }
+
+ @Test def testTableStruct {
+ val p = new ThriftParser
+ val parser = new JsonParser
+
+ var td: Option[ThriftDef] = p( """// table information
+ struct Table {
+ 1: string tableName, // name of the table
+ 2: string dbName, // database name ('default')
+ 3: string owner, // owner of this table
+ 4: i32 createTime, // creation time of the table
+ 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
+ 6: i32 retention, // retention time
+ 7: StorageDescriptor sd, // storage descriptor of the table
+ 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
+ 9: map<string, string> parameters, // to store comments or any other user level parameters
+ 10: string viewOriginalText, // original view text, null for non-view
+ 11: string viewExpandedText, // expanded view text, null for non-view
+ 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
+ 13: optional PrincipalPrivilegeSet privileges,
+ 14: optional bool temporary=false
+ }""")
+
+ val parsed = parser.parse(toJson(td.get))
+ val sample = parser.parse( """{
+ "includes":[],
+ "cppIncludes":[],
+ "namespaces":[],
+ "constants":[],
+ "typedefs":[],
+ "enums":[],
+ "senums":[],
+ "structs":[{
+ "name":"Table",
+ "xsdAll":false,
+ "fields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"tableName",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"dbName",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":3
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"owner",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":4
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"i32"
+ },
+ "name":"createTime",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":5
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"i32"
+ },
+ "name":"lastAccessTime",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":6
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"i32"
+ },
+ "name":"retention",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":7
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"StorageDescriptor"
+ },
+ "name":"sd",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":8
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "elemType":{
+ "name":"FieldSchema"
+ }
+ },
+ "name":"partitionKeys",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":9
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "keyType":{
+ "typ":"string"
+ },
+ "valueType":{
+ "typ":"string"
+ }
+ },
+ "name":"parameters",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":10
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"viewOriginalText",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":11
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"viewExpandedText",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":12
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"tableType",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":13
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"PrincipalPrivilegeSet"
+ },
+ "name":"privileges",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":14
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"bool"
+ },
+ "name":"temporary",
+ "fieldValue":{
+ "value":"false"
+ },
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ }],
+ "unions":[],
+ "xceptions":[],
+ "services":[]
+}""")
+
+ Assert.assertEquals(parsed.toString, sample.toString)
+ }
+
+ @Test def testHiveThrift {
+ val p = new ThriftParser
+ val is = getClass().getResourceAsStream("/test.thrift")
+ val src: Source = Source.fromInputStream(is)
+ val t: String = src.getLines().mkString("\n")
+ var td: Option[ThriftDef] = p(t)
+ Assert.assertTrue(td.isDefined)
+ //println(toJson(td.get))
+ }
+
+ @Test def testService {
+ val p = new ThriftParser
+ val parser = new JsonParser
+
+ var td: Option[ThriftDef] = p( """/**
+ * This interface is live.
+ */
+ service ThriftHiveMetastore extends fb303.FacebookService
+ {
+ string getMetaConf(1:string key) throws(1:MetaException o1)
+ void setMetaConf(1:string key, 2:string value) throws(1:MetaException o1)
+
+ void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
+ Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ void drop_database(1:string name, 2:bool deleteData, 3:bool cascade) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
+ list<string> get_databases(1:string pattern) throws(1:MetaException o1)
+ list<string> get_all_databases() throws(1:MetaException o1)
+ void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
+ }""")
+
+ val parsed = parser.parse(toJson(td.get))
+ val sample = parser.parse( """{
+ "includes":[],
+ "cppIncludes":[],
+ "namespaces":[],
+ "constants":[],
+ "typedefs":[],
+ "enums":[],
+ "senums":[],
+ "structs":[],
+ "unions":[],
+ "xceptions":[],
+ "services":[{
+ "name":"ThriftHiveMetastore",
+ "superName":"fb303.FacebookService",
+ "functions":[{
+ "oneway":false,
+ "returnType":{
+ "typ":"string"
+ },
+ "name":"getMetaConf",
+ "parameters":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"key",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ },{
+ "oneway":false,
+ "returnType":{
+
+ },
+ "name":"setMetaConf",
+ "parameters":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"key",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"value",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ },{
+ "oneway":false,
+ "returnType":{
+
+ },
+ "name":"create_database",
+ "parameters":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"Database"
+ },
+ "name":"database",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"AlreadyExistsException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"InvalidObjectException"
+ },
+ "name":"o2",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":3
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o3",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ },{
+ "oneway":false,
+ "returnType":{
+ "name":"Database"
+ },
+ "name":"get_database",
+ "parameters":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"name",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"NoSuchObjectException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o2",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ },{
+ "oneway":false,
+ "returnType":{
+
+ },
+ "name":"drop_database",
+ "parameters":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"name",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"bool"
+ },
+ "name":"deleteData",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":3
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"bool"
+ },
+ "name":"cascade",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"NoSuchObjectException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"InvalidOperationException"
+ },
+ "name":"o2",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":3
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o3",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ },{
+ "oneway":false,
+ "returnType":{
+ "elemType":{
+ "typ":"string"
+ }
+ },
+ "name":"get_databases",
+ "parameters":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"pattern",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ },{
+ "oneway":false,
+ "returnType":{
+ "elemType":{
+ "typ":"string"
+ }
+ },
+ "name":"get_all_databases",
+ "parameters":[],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ },{
+ "oneway":false,
+ "returnType":{
+
+ },
+ "name":"alter_database",
+ "parameters":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "typ":"string"
+ },
+ "name":"dbname",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"Database"
+ },
+ "name":"db",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }],
+ "throwFields":[{
+ "id":{
+ "value":1
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"MetaException"
+ },
+ "name":"o1",
+ "xsdOptional":false,
+ "xsdNillable":false
+ },{
+ "id":{
+ "value":2
+ },
+ "requiredNess":false,
+ "fieldType":{
+ "name":"NoSuchObjectException"
+ },
+ "name":"o2",
+ "xsdOptional":false,
+ "xsdNillable":false
+ }]
+ }]
+ }]
+}""")
+
+ Assert.assertEquals(parsed.toString, sample.toString)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala b/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
deleted file mode 100755
index fdab63e..0000000
--- a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.dsl
-
-import org.apache.hadoop.metadata.dsl._
-import org.apache.hadoop.metadata.tools.hive.HiveMockMetadataService
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil
-import org.apache.hadoop.metadata.typesystem.types.{DataTypes, StructType, TypeSystem}
-import org.json4s.native.JsonMethods._
-import org.junit.{Assert, Before, Test}
-
-/**
- * DSL Test.
- */
-class DSLTest {
- val STRUCT_TYPE_1: String = "t1"
- val STRUCT_TYPE_2: String = "t2"
-
-
- @Before
- def setup {
- val ts: TypeSystem = TypeSystem.getInstance
- ts.reset()
-
- val structType: StructType = ts.defineStructType(
- STRUCT_TYPE_1, true,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE),
- TypesUtil.createOptionalAttrDef("e", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("f", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("g", DataTypes.LONG_TYPE),
- TypesUtil.createOptionalAttrDef("h", DataTypes.FLOAT_TYPE),
- TypesUtil.createOptionalAttrDef("i", DataTypes.DOUBLE_TYPE),
- TypesUtil.createOptionalAttrDef("j", DataTypes.BIGINTEGER_TYPE),
- TypesUtil.createOptionalAttrDef("k", DataTypes.BIGDECIMAL_TYPE),
- TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
- TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- TypesUtil.createOptionalAttrDef("o",
- ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
- val recursiveStructType: StructType = ts.defineStructType(
- STRUCT_TYPE_2, true,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2))
- }
-
- @Test def test1 {
-
- // 1. Existing Types in System
- //Assert.assertEquals(s"${listTypes}", "[t2, t1, int, array<bigdecimal>, long, double, date, float, short, biginteger, byte, string, boolean, bigdecimal, map<string,double>, array<int>]")
-
- defineStructType("mytype",
- attrDef("a", INT_TYPE, ATTR_REQUIRED),
- attrDef("b", BOOLEAN_TYPE),
- attrDef("c", BYTE_TYPE),
- attrDef("d", SHORT_TYPE),
- attrDef("e", INT_TYPE),
- attrDef("f", INT_TYPE),
- attrDef("g", LONG_TYPE),
- attrDef("h", FLOAT_TYPE),
- attrDef("i", DOUBLE_TYPE),
- attrDef("j", BIGINT_TYPE),
- attrDef("k", BIGDECIMAL_TYPE),
- attrDef("l", DATE_TYPE),
- attrDef("m", arrayType(INT_TYPE)),
- attrDef("n", arrayType(BIGDECIMAL_TYPE)),
- attrDef("o", mapType(STRING_TYPE, DOUBLE_TYPE)))
-
- // 2. 'mytype' available as a a Type
- Assert.assertEquals(s"${listTypes}", "[array<bigdecimal>,array<int>,map<string,double>,mytype,t1,t2]")
-
- // 3. Create a 'mytype' instance from Json
- val i = createInstance("mytype", """
- {
- "$typeName$":"mytype",
- "e":1,
- "n":[1,1.1],
- "h":1.0,
- "b":true,
- "k":1,
- "j":1,
- "d":2,
- "m":[1,1],
- "g":1,
- "a":1,
- "i":1.0,
- "c":1,
- "l":"2014-12-03T08:00:00.000Z",
- "f":1,
- "o":{
- "b":2.0,
- "a":1.0
- }
- }
- """)
-
- // 4. Navigate mytype instance in code
- // Examples of Navigate mytype instance in code
- Assert.assertEquals(s"${i.a}", "1")
- Assert.assertEquals(s"${i.o}", "{b=2.0, a=1.0}")
- Assert.assertEquals(s"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}", "[b, a]")
-
- // 5. Serialize mytype instance to Json
- Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03T08:00:00.000Z\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}")
- }
-
- @Test def test2 {
-
- // 1. Existing Types in System
- Assert.assertEquals(s"${listTypes}", "[array<bigdecimal>,array<int>,map<string,double>,t1,t2]")
-
- val addrType = defineStructType("addressType",
- attrDef("houseNum", INT_TYPE, ATTR_REQUIRED),
- attrDef("street", STRING_TYPE, ATTR_REQUIRED),
- attrDef("city", STRING_TYPE, ATTR_REQUIRED),
- attrDef("state", STRING_TYPE, ATTR_REQUIRED),
- attrDef("zip", INT_TYPE, ATTR_REQUIRED),
- attrDef("country", STRING_TYPE, ATTR_REQUIRED)
- )
-
- val personType = defineStructType("personType",
- attrDef("first_name", STRING_TYPE, ATTR_REQUIRED),
- attrDef("last_name", STRING_TYPE, ATTR_REQUIRED),
- attrDef("address", addrType)
- )
-
- // 2. updated Types in System
- Assert.assertEquals(s"${listTypes}", "[addressType,array<bigdecimal>,array<int>,map<string,double>,personType,t1,t2]")
-
-
- // 3. Construct a Person in Code
- val person = createInstance("personType")
- val address = createInstance("addressType")
-
- person.first_name = "Meta"
- person.last_name = "Hadoop"
-
- address.houseNum = 3460
- address.street = "W Bayshore Rd"
- address.city = "Palo Alto"
- address.state = "CA"
- address.zip = 94303
- address.country = "USA"
-
- person.address = address
-
- // 4. Convert to Json
- Assert.assertEquals(s"${pretty(render(person))}", "{\n \"$typeName$\":\"personType\",\n \"first_name\":\"Meta\",\n \"address\":{\n \"$typeName$\":\"addressType\",\n \"houseNum\":3460,\n \"city\":\"Palo Alto\",\n \"country\":\"USA\",\n \"state\":\"CA\",\n \"zip\":94303,\n \"street\":\"W Bayshore Rd\"\n },\n \"last_name\":\"Hadoop\"\n}");
-
- val p2 = createInstance("personType", """{
- "first_name":"Meta",
- "address":{
- "houseNum":3460,
- "city":"Palo Alto",
- "country":"USA",
- "state":"CA",
- "zip":94303,
- "street":"W Bayshore Rd"
- },
- "last_name":"Hadoop"
- }""")
-
- }
-
- @Test def testHive(): Unit = {
- val hiveTable = HiveMockMetadataService.getTable("tpcds", "date_dim")
- //println(hiveTable)
-
- //name : String, typeName : String, comment : String
- val fieldType = defineStructType("FieldSchema",
- attrDef("name", STRING_TYPE, ATTR_REQUIRED),
- attrDef("typeName", STRING_TYPE, ATTR_REQUIRED),
- attrDef("comment", STRING_TYPE)
- )
- /*
- SerDe(name : String, serializationLib : String, parameters : Map[String, String])
- */
- defineStructType("SerDe",
- attrDef("name", STRING_TYPE, ATTR_REQUIRED),
- attrDef("serializationLib", STRING_TYPE, ATTR_REQUIRED),
- attrDef("parameters", mapType(STRING_TYPE, STRING_TYPE))
- )
-
- /*
- StorageDescriptor(fields : List[FieldSchema],
- location : String, inputFormat : String,
- outputFormat : String, compressed : Boolean,
- numBuckets : Int, bucketColumns : List[String],
- sortColumns : List[String],
- parameters : Map[String, String],
- storedAsSubDirs : Boolean
- )
- */
- val sdType = defineStructType("StorageDescriptor",
- attrDef("location", STRING_TYPE, ATTR_REQUIRED),
- attrDef("inputFormat", STRING_TYPE, ATTR_REQUIRED),
- attrDef("outputFormat", STRING_TYPE, ATTR_REQUIRED),
- attrDef("compressed", BOOLEAN_TYPE),
- attrDef("numBuckets", INT_TYPE),
- attrDef("bucketColumns", arrayType(STRING_TYPE)),
- attrDef("sortColumns", arrayType(STRING_TYPE)),
- attrDef("parameters", mapType(STRING_TYPE, STRING_TYPE)),
- attrDef("storedAsSubDirs", BOOLEAN_TYPE)
- )
-
- /*
- case class Table(dbName : String, tableName : String, storageDesc : StorageDescriptor,
- parameters : Map[String, String],
- tableType : String)
- */
- defineStructType("Table",
- attrDef("dbName", STRING_TYPE, ATTR_REQUIRED),
- attrDef("tableName", STRING_TYPE, ATTR_REQUIRED),
- attrDef("storageDesc", sdType, ATTR_REQUIRED),
- attrDef("compressed", BOOLEAN_TYPE),
- attrDef("numBuckets", INT_TYPE),
- attrDef("bucketColumns", arrayType(STRING_TYPE)),
- attrDef("sortColumns", arrayType(STRING_TYPE)),
- attrDef("parameters", mapType(STRING_TYPE, STRING_TYPE)),
- attrDef("storedAsSubDirs", BOOLEAN_TYPE)
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/tools/src/test/scala/org/apache/hadoop/metadata/tools/hive/HiveMockMetadataService.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/hadoop/metadata/tools/hive/HiveMockMetadataService.scala b/tools/src/test/scala/org/apache/hadoop/metadata/tools/hive/HiveMockMetadataService.scala
deleted file mode 100755
index 3a8215d..0000000
--- a/tools/src/test/scala/org/apache/hadoop/metadata/tools/hive/HiveMockMetadataService.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.tools.hive
-
-object HiveMockMetadataService {
-
- def getTable(dbName: String, table: String): Table = {
- return Table(dbName, table,
- StorageDescriptor(
- List[FieldSchema](
- FieldSchema("d_date_sk", "int", null),
- FieldSchema("d_date_id", "string", null),
- FieldSchema("d_date", "string", null),
- FieldSchema("d_month_seq", "int", null),
- FieldSchema("d_week_seq", "int", null),
- FieldSchema("d_quarter_seq", "int", null),
- FieldSchema("d_year", "int", null),
- FieldSchema("d_dow", "int", null),
- FieldSchema("d_moy", "int", null),
- FieldSchema("d_dom", "int", null),
- FieldSchema("d_qoy", "int", null),
- FieldSchema("d_fy_year", "int", null),
- FieldSchema("d_fy_quarter_seq", "int", null),
- FieldSchema("d_fy_week_seq", "int", null),
- FieldSchema("d_day_name", "string", null),
- FieldSchema("d_quarter_name", "string", null),
- FieldSchema("d_holiday", "string", null),
- FieldSchema("d_weekend", "string", null),
- FieldSchema("d_following_holiday", "string", null),
- FieldSchema("d_first_dom", "int", null),
- FieldSchema("d_last_dom", "int", null),
- FieldSchema("d_same_day_ly", "int", null),
- FieldSchema("d_same_day_lq", "int", null),
- FieldSchema("d_current_day", "string", null),
- FieldSchema("d_current_week", "string", null),
- FieldSchema("d_current_month", "string", null),
- FieldSchema("d_current_quarter", "string", null),
- FieldSchema("d_current_year", "string", null)
- ),
- "file:/tmp/warehouse/tpcds.db/date_dim",
- "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
- "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
- false,
- 0, List[String](), List[String](),
- Map[String, String](),
- false
- ),
- Map[String, String](),
- "Table")
- }
-
- case class FieldSchema(name: String, typeName: String, comment: String)
-
- case class SerDe(name: String, serializationLib: String, parameters: Map[String, String])
-
- case class StorageDescriptor(fields: List[FieldSchema],
- location: String, inputFormat: String,
- outputFormat: String, compressed: Boolean,
- numBuckets: Int, bucketColumns: List[String],
- sortColumns: List[String],
- parameters: Map[String, String],
- storedAsSubDirs: Boolean
- )
-
- case class Table(dbName: String, tableName: String, storageDesc: StorageDescriptor,
- parameters: Map[String, String],
- tableType: String)
-}