You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ve...@apache.org on 2015/06/17 01:04:36 UTC
[02/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java
deleted file mode 100755
index 1ae733f..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/CredentialProviderUtilityIT.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metadata.util.CredentialProviderUtility;
-import org.apache.hadoop.metadata.web.service.SecureEmbeddedServer;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.*;
-
-import static org.apache.hadoop.metadata.security.SecurityProperties.*;
-
-/**
- *
- */
-public class CredentialProviderUtilityIT {
-
- private char[] defaultPass = new char[]{'k', 'e', 'y', 'p', 'a', 's', 's'};
-
- @Test
- public void testEnterValidValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- return defaultPass;
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- }
-
- protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry) {
- assertCredentialEntryCorrect(entry, defaultPass);
- }
-
- protected void assertCredentialEntryCorrect(CredentialProvider.CredentialEntry entry, char[] password) {
- Assert.assertNotNull(entry);
- Assert.assertEquals(entry.getCredential(), password);
- }
-
- @Test
- public void testEnterEmptyValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
-
- private Random random = new Random();
-
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- List<char[]> responses = new ArrayList<>();
- responses.add(new char[0]);
- responses.add(defaultPass);
-
- int size = responses.size();
- int item = random.nextInt(size);
- return responses.get(item);
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- }
-
- @Test
- public void testEnterMismatchedValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
-
- int i = 0;
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- List<char[]> responses = new ArrayList<>();
- responses.add(defaultPass);
- responses.add(new char[] {'b', 'a', 'd', 'p', 'a', 's', 's'});
- responses.add(defaultPass);
-
- int item = i % 3;
- i++;
- return responses.get(item);
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry);
- }
-
- @Test
- public void testOverwriteValues() throws Exception {
- Path testPath = null;
- try {
- testPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks");
- } catch (IOException e) {
- e.printStackTrace();
- }
- new File(testPath.toUri().getPath()).delete();
- final Path finalTestPath = testPath;
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return finalTestPath.toString();
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- return defaultPass;
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- // now attempt to overwrite values
- CredentialProviderUtility.textDevice = new CredentialProviderUtility.TextDevice() {
-
- int i = 0;
-
- @Override
- public void printf(String fmt, Object... params) {
- System.out.print(String.format(fmt, params));
- }
-
- public String readLine(String fmt, Object ... args) {
- return i++ == 0 ? finalTestPath.toString() : "y";
- }
-
- @Override
- public char[] readPassword(String fmt, Object ... args) {
- return new char[] {'n', 'e', 'w', 'p', 'a', 's', 's'};
- }
- };
-
- CredentialProviderUtility.main(new String[] {});
-
- String providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + testPath.toUri();
- Configuration conf = new Configuration(false);
-
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl);
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- char[] newpass = "newpass".toCharArray();
- CredentialProvider.CredentialEntry entry =
- provider.getCredentialEntry(KEYSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry, newpass);
- entry = provider.getCredentialEntry(TRUSTSTORE_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry, newpass);
- entry = provider.getCredentialEntry(SERVER_CERT_PASSWORD_KEY);
- assertCredentialEntryCorrect(entry, newpass);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java
deleted file mode 100644
index 857a42a..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationKerberosFilterIT.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.filters;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdfs.web.URLConnectionFactory;
-import org.apache.hadoop.metadata.security.BaseSecurityTest;
-import org.apache.hadoop.metadata.web.service.EmbeddedServer;
-import org.mortbay.jetty.Server;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import javax.security.auth.Subject;
-import javax.security.auth.callback.*;
-import javax.security.auth.login.LoginContext;
-import javax.security.auth.login.LoginException;
-import java.io.File;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.security.PrivilegedExceptionAction;
-import java.util.Properties;
-
-/**
- *
- */
-public class MetadataAuthenticationKerberosFilterIT extends BaseSecurityTest {
- public static final String TEST_USER_JAAS_SECTION = "TestUser";
- public static final String TESTUSER = "testuser";
- public static final String TESTPASS = "testpass";
-
- private File userKeytabFile;
- private File httpKeytabFile;
-
- class TestEmbeddedServer extends EmbeddedServer {
- public TestEmbeddedServer(int port, String path) throws IOException {
- super(port, path);
- }
-
- Server getServer() {
- return server;
- }
- }
-
- @Test
- public void testKerberosBasedLogin() throws Exception {
- String originalConf = System.getProperty("metadata.conf");
- System.setProperty("metadata.conf", System.getProperty("user.dir"));
-
- setupKDCAndPrincipals();
- TestEmbeddedServer server = null;
-
- try {
- // setup the application.properties file
- generateKerberosTestProperties();
-
- // need to create the web application programmatically in order to control the injection of the test
- // application properties
- server = new TestEmbeddedServer(23000, "webapp/target/metadata-governance");
-
- startEmbeddedServer(server.getServer());
-
- final URLConnectionFactory connectionFactory = URLConnectionFactory.DEFAULT_SYSTEM_CONNECTION_FACTORY;
- // attempt to hit server and get rejected
- URL url = new URL("http://localhost:23000/");
- HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, false);
- connection.setRequestMethod("GET");
- connection.connect();
-
- Assert.assertEquals(connection.getResponseCode(), 401);
-
- // need to populate the ticket cache with a local user, so logging in...
- Subject subject = loginTestUser();
-
- Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- // attempt to hit server and get rejected
- URL url = new URL("http://localhost:23000/");
- HttpURLConnection connection = (HttpURLConnection) connectionFactory.openConnection(url, true);
- connection.setRequestMethod("GET");
- connection.connect();
-
- Assert.assertEquals(connection.getResponseCode(), 200);
-
- return null;
- }
- });
- } finally {
- server.getServer().stop();
- kdc.stop();
-
- if (originalConf != null) {
- System.setProperty("metadata.conf", originalConf);
- } else {
- System.clearProperty("metadata.conf");
- }
-
- }
-
-
- }
-
- protected Subject loginTestUser() throws LoginException, IOException {
- LoginContext lc = new LoginContext(TEST_USER_JAAS_SECTION, new CallbackHandler() {
-
- @Override
- public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
- for (int i = 0; i < callbacks.length; i++) {
- if (callbacks[i] instanceof PasswordCallback) {
- PasswordCallback passwordCallback = (PasswordCallback) callbacks[i];
- passwordCallback.setPassword(TESTPASS.toCharArray());
- }
- if (callbacks[i] instanceof NameCallback) {
- NameCallback nameCallback = (NameCallback) callbacks[i];
- nameCallback.setName(TESTUSER);
- }
- }
- }
- });
- // attempt authentication
- lc.login();
- return lc.getSubject();
- }
-
- protected void generateKerberosTestProperties() throws IOException, ConfigurationException {
- Properties props = new Properties();
- props.setProperty("metadata.http.authentication.enabled", "true");
- props.setProperty("metadata.http.authentication.type", "kerberos");
- props.setProperty("metadata.http.authentication.kerberos.principal", "HTTP/localhost@" + kdc.getRealm());
- props.setProperty("metadata.http.authentication.kerberos.keytab", httpKeytabFile.getAbsolutePath());
- props.setProperty("metadata.http.authentication.kerberos.name.rules",
- "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT");
-
- generateTestProperties(props);
- }
-
- public void setupKDCAndPrincipals() throws Exception {
- // set up the KDC
- File kdcWorkDir = startKDC();
-
- userKeytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
- httpKeytabFile = createKeytab(kdc, kdcWorkDir, "HTTP", "spnego.service.keytab");
-
- // create a test user principal
- kdc.createPrincipal(TESTUSER, TESTPASS);
-
- StringBuilder jaas = new StringBuilder(1024);
- jaas.append("TestUser {\n" +
- " com.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true;\n" +
- "};\n");
- jaas.append(createJAASEntry("Client", "dgi", userKeytabFile));
- jaas.append(createJAASEntry("Server", "HTTP", httpKeytabFile));
-
- File jaasFile = new File(kdcWorkDir, "jaas.txt");
- FileUtils.write(jaasFile, jaas.toString());
- bindJVMtoJAASFile(jaasFile);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java
deleted file mode 100644
index f41ad0a..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/filters/MetadataAuthenticationSimpleFilterIT.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.filters;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.hadoop.metadata.security.BaseSecurityTest;
-import org.apache.hadoop.metadata.web.service.EmbeddedServer;
-import org.mortbay.jetty.Server;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Properties;
-
-/**
- *
- */
-public class MetadataAuthenticationSimpleFilterIT extends BaseSecurityTest {
-
- class TestEmbeddedServer extends EmbeddedServer {
- public TestEmbeddedServer(int port, String path) throws IOException {
- super(port, path);
- }
-
- Server getServer() {
- return server;
- }
- }
-
- @Test
- public void testSimpleLogin() throws Exception {
- String originalConf = System.getProperty("metadata.conf");
- System.setProperty("metadata.conf", System.getProperty("user.dir"));
- generateSimpleLoginConfiguration();
-
- TestEmbeddedServer server = new TestEmbeddedServer(23001, "webapp/target/metadata-governance");
-
- try {
- startEmbeddedServer(server.getServer());
-
- URL url = new URL("http://localhost:23001");
- HttpURLConnection connection = (HttpURLConnection)url.openConnection();
- connection.setRequestMethod("GET");
- connection.connect();
-
- try {
- Assert.assertEquals(connection.getResponseCode(), 403);
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- url = new URL("http://localhost:23001/?user.name=testuser");
- connection = (HttpURLConnection)url.openConnection();
- connection.setRequestMethod("GET");
- connection.connect();
-
- Assert.assertEquals(connection.getResponseCode(), 200);
- } finally {
- server.getServer().stop();
- if (originalConf != null) {
- System.setProperty("metadata.conf", originalConf);
- } else {
- System.clearProperty("metadata.conf");
- }
- }
-
-
- }
-
- protected void generateSimpleLoginConfiguration() throws IOException, ConfigurationException {
- Properties config = new Properties();
- config.setProperty("metadata.http.authentication.enabled", "true");
- config.setProperty("metadata.http.authentication.type", "simple");
-
- generateTestProperties(config);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java
deleted file mode 100644
index be7171b..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/listeners/LoginProcessorIT.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.metadata.web.listeners;
-
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.metadata.security.BaseSecurityTest;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-
-/**
- *
- */
-public class LoginProcessorIT extends BaseSecurityTest {
-
- protected static final String kerberosRule =
- "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT";
-
- @Test
- public void testDefaultSimpleLogin() throws Exception {
- LoginProcessor processor = new LoginProcessor() {
- @Override
- protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
- return new PropertiesConfiguration();
- }
- };
- processor.login();
-
- Assert.assertNotNull(UserGroupInformation.getCurrentUser());
- Assert.assertFalse(UserGroupInformation.isLoginKeytabBased());
- Assert.assertFalse(UserGroupInformation.isSecurityEnabled());
- }
-
- @Test
- public void testKerberosLogin() throws Exception {
- final File keytab = setupKDCAndPrincipals();
-
- LoginProcessor processor = new LoginProcessor() {
- @Override
- protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException {
- PropertiesConfiguration config = new PropertiesConfiguration();
- config.setProperty("metadata.authentication.method", "kerberos");
- config.setProperty("metadata.authentication.principal", "dgi@EXAMPLE.COM");
- config.setProperty("metadata.authentication.keytab", keytab.getAbsolutePath());
- return config;
- }
-
- @Override
- protected Configuration getHadoopConfiguration() {
- Configuration config = new Configuration(false);
- config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- config.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
- config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, kerberosRule);
-
- return config;
- }
-
- @Override
- protected boolean isHadoopCluster() {
- return true;
- }
- };
- processor.login();
-
- Assert.assertTrue(UserGroupInformation.getLoginUser().getShortUserName().endsWith("dgi"));
- Assert.assertNotNull(UserGroupInformation.getCurrentUser());
- Assert.assertTrue(UserGroupInformation.isLoginKeytabBased());
- Assert.assertTrue(UserGroupInformation.isSecurityEnabled());
-
- kdc.stop();
-
- }
-
- private File setupKDCAndPrincipals() throws Exception {
- // set up the KDC
- File kdcWorkDir = startKDC();
-
- Assert.assertNotNull(kdc.getRealm());
-
- File keytabFile = createKeytab(kdc, kdcWorkDir, "dgi", "dgi.keytab");
-
- return keytabFile;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java
deleted file mode 100755
index ca51cd6..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/AdminJerseyResourceIT.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-
-/**
- * Integration test for Admin jersey resource.
- */
-public class AdminJerseyResourceIT extends BaseResourceIT {
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
- }
-
- @Test
- public void testGetVersion() throws Exception {
- WebResource resource = service
- .path("api/metadata/admin/version");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- PropertiesConfiguration buildConfiguration =
- new PropertiesConfiguration("metadata-buildinfo.properties");
-
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertEquals(response.get("Version"),
- buildConfiguration.getString("build.version"));
- Assert.assertEquals(response.get("Name"),
- buildConfiguration.getString("project.name"));
- Assert.assertEquals(response.get("Description"),
- buildConfiguration.getString("project.description"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java
deleted file mode 100755
index e0efe7d..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/BaseResourceIT.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriBuilder;
-
-/**
- * Base class for integration tests.
- * Sets up the web resource and has helper methods to create type and entity.
- */
-public abstract class BaseResourceIT {
-
- protected WebResource service;
- protected MetadataServiceClient serviceClient;
- public static String baseUrl = "http://localhost:21000/";
-
- @BeforeClass
- public void setUp() throws Exception {
-
- DefaultClientConfig config = new DefaultClientConfig();
- Client client = Client.create(config);
- client.resource(UriBuilder.fromUri(baseUrl).build());
-
- service = client.resource(UriBuilder.fromUri(baseUrl).build());
- serviceClient = new MetadataServiceClient(baseUrl);
- }
-
- protected void createType(TypesDef typesDef) throws Exception {
- HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0);
- if (serviceClient.getType(sampleType.typeName) == null ) {
- String typesAsJSON = TypesSerialization.toJson(typesDef);
- createType(typesAsJSON);
- }
- }
-
- protected void createType(String typesAsJSON) throws Exception {
- WebResource resource = service
- .path("api/metadata/types");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, typesAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get("types"));
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- }
-
- protected Id createInstance(Referenceable referenceable) throws Exception {
- String typeName = referenceable.getTypeName();
- System.out.println("creating instance of type " + typeName);
-
- String entityJSON = InstanceSerialization.toJson(referenceable, true);
- System.out.println("Submitting new entity= " + entityJSON);
- JSONObject jsonObject = serviceClient.createEntity(entityJSON);
- String guid = jsonObject.getString(MetadataServiceClient.GUID);
- System.out.println("created instance for type " + typeName + ", guid: " + guid);
-
- // return the reference to created instance with guid
- return new Id(guid, 0, referenceable.getTypeName());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
deleted file mode 100755
index 00cc234..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/EntityJerseyResourceIT.java
+++ /dev/null
@@ -1,679 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.collect.ImmutableList;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.MetadataServiceException;
-import org.apache.hadoop.metadata.typesystem.IStruct;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.Struct;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization;
-import org.apache.hadoop.metadata.typesystem.json.InstanceSerialization$;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization;
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization$;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.EnumValue;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * Integration tests for Entity Jersey Resource.
- */
-public class EntityJerseyResourceIT extends BaseResourceIT {
-
- private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class);
-
- private static final String DATABASE_TYPE = "hive_database";
- private static final String DATABASE_NAME = "foo";
- private static final String TABLE_TYPE = "hive_table_type";
- private static final String TABLE_NAME = "bar";
- private static final String TRAITS = "traits";
-
- private Referenceable tableInstance;
- private Id tableId;
- private String traitName;
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
-
- createHiveTypes();
- }
-
- @Test
- public void testSubmitEntity() throws Exception {
- tableInstance = createHiveTableInstance();
- tableId = createInstance(tableInstance);
-
- final String guid = tableId._getId();
- try {
- Assert.assertNotNull(UUID.fromString(guid));
- } catch (IllegalArgumentException e) {
- Assert.fail("Response is not a guid, " + guid);
- }
- }
-
- @DataProvider
- public Object[][] invalidAttrValues() {
- return new Object[][]{
- {null}, {""}, {" "}};
- }
-
- @Test(dataProvider = "invalidAttrValues")
- public void testEntityInvalidValue(String value) throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", randomString());
- databaseInstance.set("description", value);
-
- try {
- createInstance(databaseInstance);
- Assert.fail("Exptected MetadataServiceException");
- } catch(MetadataServiceException e) {
- Assert.assertEquals(e.getStatus(), ClientResponse.Status.BAD_REQUEST);
- }
- }
-
- @Test
- public void testSubmitEntityWithBadDateFormat() throws Exception {
-
- try {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", DATABASE_NAME);
- databaseInstance.set("description", "foo database");
-
- Referenceable tableInstance = new Referenceable(TABLE_TYPE,
- "classification", "pii", "phi", "pci", "sox", "sec", "finance");
- tableInstance.set("name", TABLE_NAME);
- tableInstance.set("description", "bar table");
- tableInstance.set("date", "2014-07-11");
- tableInstance.set("type", "managed");
- tableInstance.set("level", 2);
- tableInstance.set("tableType", 1); // enum
- tableInstance.set("database", databaseInstance);
- tableInstance.set("compressed", false);
-
- Struct traitInstance = (Struct) tableInstance.getTrait("classification");
- traitInstance.set("tag", "foundation_etl");
-
- Struct serde1Instance = new Struct("serdeType");
- serde1Instance.set("name", "serde1");
- serde1Instance.set("serde", "serde1");
- tableInstance.set("serde1", serde1Instance);
-
- Struct serde2Instance = new Struct("serdeType");
- serde2Instance.set("name", "serde2");
- serde2Instance.set("serde", "serde2");
- tableInstance.set("serde2", serde2Instance);
-
- tableId = createInstance(tableInstance);
- Assert.fail("Was expecting an exception here ");
- } catch (MetadataServiceException e) {
- Assert.assertTrue(
- e.getMessage().contains("\"error\":\"Cannot convert value '2014-07-11' to datatype date\""));
- }
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testAddProperty() throws Exception {
- final String guid = tableId._getId();
- //add property
- String description = "bar table - new desc";
- ClientResponse clientResponse = addProperty(guid, "description", description);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String entityRef = getEntityDefinition(getEntityDefinition(guid));
- Assert.assertNotNull(entityRef);
-
- tableInstance.set("description", description);
-
- //invalid property for the type
- clientResponse = addProperty(guid, "invalid_property", "bar table");
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
-
- //non-string property, update
- clientResponse = addProperty(guid, "level", "4");
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- entityRef = getEntityDefinition(getEntityDefinition(guid));
- Assert.assertNotNull(entityRef);
-
- tableInstance.set("level", 4);
- }
-
- @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
- public void testAddNullProperty() throws Exception {
- final String guid = tableId._getId();
- //add property
- addProperty(guid, null, "foo bar");
- Assert.fail();
- }
-
- @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
- public void testAddNullPropertyValue() throws Exception {
- final String guid = tableId._getId();
- //add property
- addProperty(guid, "description", null);
- Assert.fail();
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testAddReferenceProperty() throws Exception {
- //Create new db instance
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", "newdb");
- databaseInstance.set("description", "new database");
-
- Id dbInstance = createInstance(databaseInstance);
- String dbId = dbInstance._getId();
-
- //Add reference property
- final String guid = tableId._getId();
- ClientResponse clientResponse = addProperty(guid, "database", dbId);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testGetEntityDefinition() throws Exception {
- final String guid = tableId._getId();
- ClientResponse clientResponse = getEntityDefinition(guid);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final String definition = response.getString(MetadataServiceClient.DEFINITION);
- Assert.assertNotNull(definition);
- LOG.debug("tableInstanceAfterGet = " + definition);
- InstanceSerialization.fromJsonReferenceable(definition, true);
- }
-
- private ClientResponse addProperty(String guid, String property, String value) {
- WebResource resource = service
- .path("api/metadata/entities")
- .path(guid);
-
- return resource.queryParam("property", property).queryParam("value", value)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.PUT, ClientResponse.class);
- }
-
- private ClientResponse getEntityDefinition(String guid) {
- WebResource resource = service
- .path("api/metadata/entities")
- .path(guid);
- return resource.accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- }
-
- private String getEntityDefinition(ClientResponse clientResponse) throws Exception {
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
- JSONObject response = new JSONObject(clientResponse.getEntity(String.class));
- final String definition = response.getString(MetadataServiceClient.DEFINITION);
- Assert.assertNotNull(definition);
-
- return definition;
- }
-
- @Test
- public void testGetInvalidEntityDefinition() throws Exception {
- WebResource resource = service
- .path("api/metadata/entities")
- .path("blah");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
- Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testGetEntityList() throws Exception {
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .queryParam("type", TABLE_TYPE)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(list);
- Assert.assertEquals(list.length(), 1);
- }
-
- @Test
- public void testGetEntityListForBadEntityType() throws Exception {
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .queryParam("type", "blah")
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
- Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
- }
-
-
- @Test
- public void testGetEntityListForNoInstances() throws Exception {
- addNewType();
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .queryParam("type", "test")
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertEquals(list.length(), 0);
- }
-
- private void addNewType() throws Exception {
- HierarchicalTypeDefinition<ClassType> testTypeDefinition =
- TypesUtil.createClassTypeDef("test", ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
- String typesAsJSON = TypesSerialization.toJson(testTypeDefinition);
- createType(typesAsJSON);
- }
-
- @Test(dependsOnMethods = "testSubmitEntity")
- public void testGetTraitNames() throws Exception {
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get("GUID"));
-
- final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS);
- Assert.assertEquals(list.length(), 7);
- }
-
- @Test(dependsOnMethods = "testGetTraitNames")
- public void testAddTrait() throws Exception {
- traitName = "PII_Trait" + randomString();
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
- String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
- LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
- createType(traitDefinitionAsJSON);
-
- Struct traitInstance = new Struct(traitName);
- String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get(MetadataServiceClient.GUID));
- }
-
- @Test(dependsOnMethods = "testAddTrait")
- public void testAddExistingTrait() throws Exception {
- final String traitName = "PII_Trait" + randomString();
-
- Struct traitInstance = new Struct(traitName);
- String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
- }
-
- @Test(dependsOnMethods = "testGetTraitNames")
- public void testAddTraitWithAttribute() throws Exception {
- final String traitName = "PII_Trait" + randomString();
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
- String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
- LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
- createType(traitDefinitionAsJSON);
-
- Struct traitInstance = new Struct(traitName);
- traitInstance.set("type", "SSN");
- String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- final String guid = tableId._getId();
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get(MetadataServiceClient.GUID));
-
- // verify the response
- clientResponse = getEntityDefinition(guid);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
- responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- final String definition = response.getString(MetadataServiceClient.DEFINITION);
- Assert.assertNotNull(definition);
- Referenceable entityRef = InstanceSerialization.fromJsonReferenceable(definition, true);
- IStruct traitRef = entityRef.getTrait(traitName);
- String type = (String) traitRef.get("type");
- Assert.assertEquals(type, "SSN");
- }
-
- @Test
- public void testAddTraitWithNoRegistration() throws Exception {
- final String traitName = "PII_Trait" + randomString();
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
- String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
- LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
-
- Struct traitInstance = new Struct(traitName);
- String traitInstanceAsJSON = InstanceSerialization$.MODULE$.toJson(traitInstance, true);
- LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path("random")
- .path(TRAITS)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
- }
-
- @Test(dependsOnMethods = "testAddTrait")
- public void testDeleteTrait() throws Exception {
- final String guid = tableId._getId();
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path(guid)
- .path(TRAITS)
- .path(traitName)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.DELETE, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
- Assert.assertNotNull(response.get("GUID"));
- Assert.assertNotNull(response.get("traitName"));
- }
-
- @Test
- public void testDeleteTraitNonExistent() throws Exception {
- final String traitName = "blah_trait";
-
- ClientResponse clientResponse = service
- .path("api/metadata/entities")
- .path("random")
- .path(TRAITS)
- .path(traitName)
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.DELETE, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.ERROR));
- Assert.assertEquals(response.getString(MetadataServiceClient.ERROR),
- "trait=" + traitName + " should be defined in type system before it can be deleted");
- Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE));
- }
-
- private String random() {
- return RandomStringUtils.random(10);
- }
-
- private String randomString() {
- return RandomStringUtils.randomAlphanumeric(10);
- }
-
- @Test
- public void testUTF8() throws Exception {
- String classType = random();
- String attrName = random();
- String attrValue = random();
-
- HierarchicalTypeDefinition<ClassType> classTypeDefinition =
- TypesUtil.createClassTypeDef(classType, ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE));
- TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
- ImmutableList.<StructTypeDefinition>of(), ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.of(classTypeDefinition));
- createType(typesDef);
-
- Referenceable instance = new Referenceable(classType);
- instance.set(attrName, attrValue);
- Id guid = createInstance(instance);
-
- ClientResponse response = getEntityDefinition(guid._getId());
- String definition = getEntityDefinition(response);
- Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
- Assert.assertEquals(getReferenceable.get(attrName), attrValue);
- }
-
- private void createHiveTypes() throws Exception {
- HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
- TypesUtil.createClassTypeDef(DATABASE_TYPE,
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
- StructTypeDefinition structTypeDefinition =
- new StructTypeDefinition("serdeType",
- new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)
- });
-
- EnumValue values[] = {
- new EnumValue("MANAGED", 1),
- new EnumValue("EXTERNAL", 2),
- };
-
- EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values);
-
- HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
- TypesUtil.createClassTypeDef(TABLE_TYPE,
- ImmutableList.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("date", DataTypes.DATE_TYPE),
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE),
- new AttributeDefinition("tableType", "tableType",
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("serde1",
- "serdeType", Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("serde2",
- "serdeType", Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("database",
- DATABASE_TYPE, Multiplicity.REQUIRED, true, null),
- new AttributeDefinition("compressed",
- DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, true, null));
-
- HierarchicalTypeDefinition<TraitType> classificationTraitDefinition =
- TypesUtil.createTraitTypeDef("classification",
- ImmutableList.<String>of(),
- TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<TraitType> piiTrait =
- TypesUtil.createTraitTypeDef("pii", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> phiTrait =
- TypesUtil.createTraitTypeDef("phi", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> pciTrait =
- TypesUtil.createTraitTypeDef("pci", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> soxTrait =
- TypesUtil.createTraitTypeDef("sox", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> secTrait =
- TypesUtil.createTraitTypeDef("sec", ImmutableList.<String>of());
- HierarchicalTypeDefinition<TraitType> financeTrait =
- TypesUtil.createTraitTypeDef("finance", ImmutableList.<String>of());
-
- TypesDef typesDef = TypeUtils.getTypesDef(
- ImmutableList.of(enumTypeDefinition),
- ImmutableList.of(structTypeDefinition),
- ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait,
- soxTrait, secTrait, financeTrait),
- ImmutableList.of(databaseTypeDefinition, tableTypeDefinition));
- createType(typesDef);
- }
-
- private Referenceable createHiveTableInstance() throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
- databaseInstance.set("name", DATABASE_NAME);
- databaseInstance.set("description", "foo database");
-
- Referenceable tableInstance = new Referenceable(TABLE_TYPE,
- "classification", "pii", "phi", "pci", "sox", "sec", "finance");
- tableInstance.set("name", TABLE_NAME);
- tableInstance.set("description", "bar table");
- tableInstance.set("date", "2014-07-11T08:00:00.000Z");
- tableInstance.set("type", "managed");
- tableInstance.set("level", 2);
- tableInstance.set("tableType", 1); // enum
- tableInstance.set("database", databaseInstance);
- tableInstance.set("compressed", false);
-
- Struct traitInstance = (Struct) tableInstance.getTrait("classification");
- traitInstance.set("tag", "foundation_etl");
-
- Struct serde1Instance = new Struct("serdeType");
- serde1Instance.set("name", "serde1");
- serde1Instance.set("serde", "serde1");
- tableInstance.set("serde1", serde1Instance);
-
- Struct serde2Instance = new Struct("serdeType");
- serde2Instance.set("name", "serde2");
- serde2Instance.set("serde", "serde2");
- tableInstance.set("serde2", serde2Instance);
-
- List<String> traits = tableInstance.getTraits();
- Assert.assertEquals(traits.size(), 7);
-
- return tableInstance;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java b/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java
deleted file mode 100644
index a40f7ce..0000000
--- a/webapp/src/test/java/org/apache/hadoop/metadata/web/resources/HiveLineageJerseyResourceIT.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.web.resources;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.metadata.MetadataServiceClient;
-import org.apache.hadoop.metadata.typesystem.Referenceable;
-import org.apache.hadoop.metadata.typesystem.TypesDef;
-import org.apache.hadoop.metadata.typesystem.persistence.Id;
-import org.apache.hadoop.metadata.typesystem.types.AttributeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.ClassType;
-import org.apache.hadoop.metadata.typesystem.types.DataTypes;
-import org.apache.hadoop.metadata.typesystem.types.EnumTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.IDataType;
-import org.apache.hadoop.metadata.typesystem.types.Multiplicity;
-import org.apache.hadoop.metadata.typesystem.types.StructTypeDefinition;
-import org.apache.hadoop.metadata.typesystem.types.TraitType;
-import org.apache.hadoop.metadata.typesystem.types.TypeUtils;
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil;
-import org.apache.hadoop.metadata.web.util.Servlets;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import javax.ws.rs.HttpMethod;
-import javax.ws.rs.core.Response;
-import java.util.List;
-
-/**
- * Hive Lineage Integration Tests.
- */
-public class HiveLineageJerseyResourceIT extends BaseResourceIT {
-
- private static final String BASE_URI = "api/metadata/lineage/hive/table/";
-
- @BeforeClass
- public void setUp() throws Exception {
- super.setUp();
-
- setUpTypes();
- setupInstances();
- }
-
- @Test
- public void testInputsGraph() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("sales_fact_monthly_mv")
- .path("inputs")
- .path("graph");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- System.out.println("inputs graph = " + responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(results);
-
- JSONObject values = results.getJSONObject("values");
- Assert.assertNotNull(values);
-
- final JSONObject vertices = values.getJSONObject("vertices");
- Assert.assertEquals(vertices.length(), 4);
-
- final JSONObject edges = values.getJSONObject("edges");
- Assert.assertEquals(edges.length(), 4);
- }
-
- @Test
- public void testOutputsGraph() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("sales_fact")
- .path("outputs")
- .path("graph");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- System.out.println("outputs graph= " + responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(results);
-
- JSONObject values = results.getJSONObject("values");
- Assert.assertNotNull(values);
-
- final JSONObject vertices = values.getJSONObject("vertices");
- Assert.assertEquals(vertices.length(), 3);
-
- final JSONObject edges = values.getJSONObject("edges");
- Assert.assertEquals(edges.length(), 4);
- }
-
- @Test
- public void testSchema() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("sales_fact")
- .path("schema");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
-
- String responseAsString = clientResponse.getEntity(String.class);
- Assert.assertNotNull(responseAsString);
- System.out.println("schema = " + responseAsString);
-
- JSONObject response = new JSONObject(responseAsString);
- Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID));
-
- JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
- Assert.assertNotNull(results);
-
- JSONArray rows = results.getJSONArray("rows");
- Assert.assertEquals(rows.length(), 4);
-
- for (int index = 0; index < rows.length(); index++) {
- final JSONObject row = rows.getJSONObject(index);
- Assert.assertNotNull(row.getString("name"));
- Assert.assertNotNull(row.getString("comment"));
- Assert.assertNotNull(row.getString("dataType"));
- Assert.assertEquals(row.getString("$typeName$"), "hive_column");
- }
- }
-
- @Test
- public void testSchemaForEmptyTable() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("")
- .path("schema");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(),
- Response.Status.NOT_FOUND.getStatusCode());
- }
-
- @Test
- public void testSchemaForInvalidTable() throws Exception {
- WebResource resource = service
- .path(BASE_URI)
- .path("blah")
- .path("schema");
-
- ClientResponse clientResponse = resource
- .accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE)
- .method(HttpMethod.GET, ClientResponse.class);
- Assert.assertEquals(clientResponse.getStatus(),
- Response.Status.NOT_FOUND.getStatusCode());
- }
-
- private void setUpTypes() throws Exception {
- TypesDef typesDef = createTypeDefinitions();
- createType(typesDef);
- }
-
- private static final String DATABASE_TYPE = "hive_db";
- private static final String HIVE_TABLE_TYPE = "hive_table";
- private static final String COLUMN_TYPE = "hive_column";
- private static final String HIVE_PROCESS_TYPE = "hive_process";
-
- private TypesDef createTypeDefinitions() {
- HierarchicalTypeDefinition<ClassType> dbClsDef
- = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("description", DataTypes.STRING_TYPE),
- attrDef("locationUri", DataTypes.STRING_TYPE),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> columnClsDef =
- TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
- attrDef("name", DataTypes.STRING_TYPE),
- attrDef("dataType", DataTypes.STRING_TYPE),
- attrDef("comment", DataTypes.STRING_TYPE)
- );
-
- HierarchicalTypeDefinition<ClassType> tblClsDef =
- TypesUtil.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.INT_TYPE),
- attrDef("lastAccessTime", DataTypes.INT_TYPE),
- attrDef("tableType", DataTypes.STRING_TYPE),
- attrDef("temporary", DataTypes.BOOLEAN_TYPE),
- new AttributeDefinition("db", DATABASE_TYPE,
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("columns",
- DataTypes.arrayTypeName(COLUMN_TYPE),
- Multiplicity.COLLECTION, true, null)
- );
-
- HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
- TypesUtil.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"),
- attrDef("userName", DataTypes.STRING_TYPE),
- attrDef("startTime", DataTypes.INT_TYPE),
- attrDef("endTime", DataTypes.INT_TYPE),
- attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
- );
-
- HierarchicalTypeDefinition<TraitType> dimTraitDef =
- TypesUtil.createTraitTypeDef("Dimension", null);
-
- HierarchicalTypeDefinition<TraitType> factTraitDef =
- TypesUtil.createTraitTypeDef("Fact", null);
-
- HierarchicalTypeDefinition<TraitType> metricTraitDef =
- TypesUtil.createTraitTypeDef("Metric", null);
-
- HierarchicalTypeDefinition<TraitType> etlTraitDef =
- TypesUtil.createTraitTypeDef("ETL", null);
-
-
- HierarchicalTypeDefinition<TraitType> piiTraitDef =
- TypesUtil.createTraitTypeDef("PII", null);
-
- return TypeUtils.getTypesDef(
- ImmutableList.<EnumTypeDefinition>of(),
- ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef, piiTraitDef),
- ImmutableList.of(dbClsDef, columnClsDef, tblClsDef, loadProcessClsDef)
- );
- }
-
- AttributeDefinition attrDef(String name, IDataType dT) {
- return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
- return attrDef(name, dT, m, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT,
- Multiplicity m, boolean isComposite, String reverseAttributeName) {
- Preconditions.checkNotNull(name);
- Preconditions.checkNotNull(dT);
- return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
- }
-
- private void setupInstances() throws Exception {
- Id salesDB = database(
- "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
-
- List<Referenceable> salesFactColumns = ImmutableList.of(
- column("time_id", "int", "time id"),
- column("product_id", "int", "product id"),
- column("customer_id", "int", "customer id", "PII"),
- column("sales", "double", "product id", "Metric")
- );
-
- Id salesFact = table("sales_fact", "sales fact table",
- salesDB, "Joe", "Managed", salesFactColumns, "Fact");
-
- List<Referenceable> timeDimColumns = ImmutableList.of(
- column("time_id", "int", "time id"),
- column("dayOfYear", "int", "day Of Year"),
- column("weekDay", "int", "week Day")
- );
-
- Id timeDim = table("time_dim", "time dimension table",
- salesDB, "John Doe", "External", timeDimColumns, "Dimension");
-
- Id reportingDB = database("Reporting", "reporting database", "Jane BI",
- "hdfs://host:8000/apps/warehouse/reporting");
-
- Id salesFactDaily = table("sales_fact_daily_mv",
- "sales fact daily materialized view",
- reportingDB, "Joe BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesDaily", "John ETL",
- ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
- "create table as select ", "plan", "id", "graph",
- "ETL");
-
- Id salesFactMonthly = table("sales_fact_monthly_mv",
- "sales fact monthly materialized view",
- reportingDB, "Jane BI", "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesMonthly", "John ETL",
- ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
- "create table as select ", "plan", "id", "graph",
- "ETL");
- }
-
- Id database(String name, String description,
- String owner, String locationUri,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("locationUri", locationUri);
- referenceable.set("createTime", System.currentTimeMillis());
-
- return createInstance(referenceable);
- }
-
- Referenceable column(String name, String dataType, String comment,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("dataType", dataType);
- referenceable.set("comment", comment);
-
- return referenceable;
- }
-
- Id table(String name, String description, Id dbId,
- String owner, String tableType,
- List<Referenceable> columns,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("tableType", tableType);
- referenceable.set("createTime", System.currentTimeMillis());
- referenceable.set("lastAccessTime", System.currentTimeMillis());
- referenceable.set("retention", System.currentTimeMillis());
-
- referenceable.set("db", dbId);
- referenceable.set("columns", columns);
-
- return createInstance(referenceable);
- }
-
- Id loadProcess(String name, String user,
- List<Id> inputTables,
- List<Id> outputTables,
- String queryText, String queryPlan,
- String queryId, String queryGraph,
- String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("user", user);
- referenceable.set("startTime", System.currentTimeMillis());
- referenceable.set("endTime", System.currentTimeMillis() + 10000);
-
- referenceable.set("inputs", inputTables);
- referenceable.set("outputs", outputTables);
-
- referenceable.set("queryText", queryText);
- referenceable.set("queryPlan", queryPlan);
- referenceable.set("queryId", queryId);
- referenceable.set("queryGraph", queryGraph);
-
- return createInstance(referenceable);
- }
-}