You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/03/12 20:31:30 UTC
[5/6] ambari git commit: AMBARI-10035. Hive View: Retrieve history
from ATS (alexantonenko)
AMBARI-10035. Hive View: Retrieve history from ATS (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db973127
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db973127
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db973127
Branch: refs/heads/trunk
Commit: db97312728bf9c04b3ec7711593fecd04a7554d1
Parents: d786be4
Author: Alex Antonenko <hi...@gmail.com>
Authored: Thu Mar 12 18:59:36 2015 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Thu Mar 12 21:31:20 2015 +0200
----------------------------------------------------------------------
.../assets/javascripts/modernizr-2.6.2.min.js | 0
contrib/views/files/src/main/resources/view.xml | 13 +-
contrib/views/hive/pom.xml | 2 +-
.../apache/ambari/view/hive/BaseService.java | 20 +-
.../apache/ambari/view/hive/HelpService.java | 2 +-
.../org/apache/ambari/view/hive/TestBean.java | 6 +-
.../ambari/view/hive/client/Connection.java | 24 +-
.../view/hive/client/ConnectionFactory.java | 72 ++++
.../ambari/view/hive/client/ConnectionPool.java | 81 ----
.../view/hive/client/IConnectionFactory.java | 24 ++
.../view/hive/persistence/DataStoreStorage.java | 32 +-
.../view/hive/persistence/IStorageFactory.java | 23 ++
.../view/hive/persistence/KeyValueStorage.java | 15 +-
.../ambari/view/hive/persistence/Storage.java | 6 +-
.../persistence/utils/FilteringStrategy.java | 1 +
.../view/hive/persistence/utils/Indexed.java | 4 +-
.../utils/OnlyOwnersFilteringStrategy.java | 5 +
.../hive/persistence/utils/StorageFactory.java | 69 ++++
.../hive/persistence/utils/StorageUtil.java | 94 -----
.../hive/resources/CRUDResourceManager.java | 42 +-
.../view/hive/resources/IResourceManager.java | 37 ++
.../resources/PersonalCRUDResourceManager.java | 20 +-
.../resources/SharedCRUDResourceManager.java | 12 +-
.../resources/browser/HiveBrowserService.java | 24 +-
.../view/hive/resources/files/FileService.java | 47 ++-
.../view/hive/resources/jobs/Aggregator.java | 210 ++++++++++
.../resources/jobs/ConnectionController.java | 21 +-
.../jobs/IOperationHandleResourceManager.java | 36 ++
.../ambari/view/hive/resources/jobs/Job.java | 90 ----
.../view/hive/resources/jobs/JobController.java | 46 ---
.../resources/jobs/JobControllerFactory.java | 43 --
.../hive/resources/jobs/JobControllerImpl.java | 326 ---------------
.../view/hive/resources/jobs/JobImpl.java | 203 ---------
.../hive/resources/jobs/JobResourceManager.java | 107 -----
.../resources/jobs/JobResourceProvider.java | 12 +-
.../view/hive/resources/jobs/JobService.java | 37 +-
.../view/hive/resources/jobs/LogParser.java | 79 ++--
.../jobs/OperationHandleController.java | 22 +-
.../jobs/OperationHandleControllerFactory.java | 25 +-
.../jobs/OperationHandleResourceManager.java | 41 +-
.../resources/jobs/StoredOperationHandle.java | 12 +-
.../hive/resources/jobs/atsJobs/ATSParser.java | 139 +++++++
.../jobs/atsJobs/ATSParserFactory.java | 42 ++
.../jobs/atsJobs/ATSRequestsDelegate.java | 29 ++
.../jobs/atsJobs/ATSRequestsDelegateImpl.java | 86 ++++
.../resources/jobs/atsJobs/HiveQueryId.java | 37 ++
.../hive/resources/jobs/atsJobs/IATSParser.java | 29 ++
.../hive/resources/jobs/atsJobs/TezDagId.java | 26 ++
.../jobs/viewJobs/IJobControllerFactory.java | 23 ++
.../view/hive/resources/jobs/viewJobs/Job.java | 99 +++++
.../resources/jobs/viewJobs/JobController.java | 46 +++
.../jobs/viewJobs/JobControllerFactory.java | 42 ++
.../jobs/viewJobs/JobControllerImpl.java | 343 ++++++++++++++++
.../hive/resources/jobs/viewJobs/JobImpl.java | 226 ++++++++++
.../jobs/viewJobs/JobResourceManager.java | 107 +++++
.../resources/resources/FileResourceItem.java | 6 +-
.../resources/FileResourceResourceManager.java | 9 +-
.../resources/FileResourceResourceProvider.java | 10 +-
.../resources/FileResourceService.java | 11 +-
.../hive/resources/savedQueries/SavedQuery.java | 6 +-
.../savedQueries/SavedQueryResourceManager.java | 42 +-
.../SavedQueryResourceProvider.java | 18 +-
.../savedQueries/SavedQueryService.java | 10 +-
.../ambari/view/hive/resources/udfs/UDF.java | 6 +-
.../hive/resources/udfs/UDFResourceManager.java | 9 +-
.../resources/udfs/UDFResourceProvider.java | 11 +-
.../view/hive/resources/udfs/UDFService.java | 15 +-
.../ambari/view/hive/utils/FilePaginator.java | 6 +-
.../apache/ambari/view/hive/utils/HdfsApi.java | 45 +-
.../apache/ambari/view/hive/utils/HdfsUtil.java | 17 +-
.../view/hive/utils/SharedObjectsFactory.java | 163 ++++++++
.../hive-web/app/components/typeahead-widget.js | 2 +-
.../ui/hive-web/app/controllers/history.js | 9 +-
.../ui/hive-web/app/controllers/index.js | 56 ++-
.../app/controllers/index/history-query/logs.js | 12 +-
.../controllers/index/history-query/results.js | 5 +-
.../ui/hive-web/app/controllers/job.js | 5 +-
.../ui/hive-web/app/controllers/open-queries.js | 13 +-
.../ui/hive-web/app/controllers/queries.js | 5 +-
.../ui/hive-web/app/controllers/settings.js | 129 +++---
.../ui/hive-web/app/controllers/tez-ui.js | 22 +
.../hive-web/app/controllers/visual-explain.js | 22 +
.../ui/hive-web/app/helpers/all-uppercase.js | 25 ++
.../ui/hive-web/app/initializers/i18n.js | 7 +-
.../resources/ui/hive-web/app/models/job.js | 14 +-
.../ui/hive-web/app/routes/application.js | 4 +-
.../app/routes/index/history-query/index.js | 13 +-
.../resources/ui/hive-web/app/styles/app.scss | 100 +++--
.../templates/components/_typeahead-widget.hbs | 17 +
.../app/templates/components/popover-widget.hbs | 2 +-
.../app/templates/databases-search-results.hbs | 2 +-
.../ui/hive-web/app/templates/databases.hbs | 4 +-
.../ui/hive-web/app/templates/history.hbs | 4 +-
.../ui/hive-web/app/templates/index.hbs | 124 +++---
.../ui/hive-web/app/templates/queries.hbs | 58 +--
.../ui/hive-web/app/templates/settings.hbs | 63 +--
.../ui/hive-web/app/templates/tez-ui.hbs | 22 +
.../hive-web/app/templates/visual-explain.hbs | 22 +
.../ui/hive-web/app/utils/constants.js | 87 +++-
.../ui/hive-web/app/utils/functions.js | 19 +-
.../resources/ui/hive-web/app/views/tez-ui.js | 35 ++
.../ui/hive-web/app/views/visual-explain.js | 35 ++
.../src/main/resources/ui/hive-web/bower.json | 8 +-
.../src/main/resources/ui/hive-web/package.json | 12 +-
.../tests/integration/query-editor-test.js | 2 +-
.../tests/unit/controllers/history-test.js | 4 +-
.../tests/unit/controllers/queries-test.js | 15 -
.../tests/unit/controllers/settings-test.js | 97 +++++
.../tests/unit/controllers/tez-ui-test.js | 33 ++
.../unit/controllers/visual-explain-test.js | 33 ++
.../tests/unit/views/visual-explain-test.js | 30 ++
contrib/views/hive/src/main/resources/view.xml | 52 ++-
.../hive/resources/files/FileServiceTest.java | 7 +-
.../view/hive/resources/jobs/ATSParserTest.java | 411 +++++++++++++++++++
.../hive/resources/jobs/AggregatorTest.java | 396 ++++++++++++++++++
.../hive/resources/jobs/JobServiceTest.java | 16 +-
.../view/hive/resources/jobs/LogParserTest.java | 8 +-
.../resources/FileResourceServiceTest.java | 4 +-
.../savedQueries/SavedQueryServiceTest.java | 16 +-
.../hive/resources/udfs/UDFServiceTest.java | 4 +-
120 files changed, 4083 insertions(+), 1645 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/files/src/main/resources/ui/app/assets/javascripts/modernizr-2.6.2.min.js
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/assets/javascripts/modernizr-2.6.2.min.js b/contrib/views/files/src/main/resources/ui/app/assets/javascripts/modernizr-2.6.2.min.js
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/files/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/view.xml b/contrib/views/files/src/main/resources/view.xml
index 206128d..eaf1a02 100644
--- a/contrib/views/files/src/main/resources/view.xml
+++ b/contrib/views/files/src/main/resources/view.xml
@@ -19,19 +19,26 @@
<label>Files</label>
<version>0.1.0</version>
+ <min-ambari-version>1.7.*</min-ambari-version>
+
<parameter>
<name>webhdfs.url</name>
- <description>WebHDFS FileSystem URI (example: webhdfs://namenode:50070)</description>
+ <description>Enter the WebHDFS FileSystem URI. Typically this is the dfs.namenode.http-address property in the hdfs-site.xml configuration. URL must be accessible from Ambari Server.</description>
+ <label>WebHDFS FileSystem URI</label>
+ <placeholder>webhdfs://namenode:50070</placeholder>
<required>true</required>
</parameter>
<parameter>
<name>webhdfs.username</name>
- <description>doAs for proxy user for HDFS</description>
+ <description>doAs for proxy user for HDFS. By default, uses the currently logged-in Ambari user.</description>
+ <label>WebHDFS Username</label>
<required>false</required>
</parameter>
<parameter>
<name>webhdfs.auth</name>
- <description>Semicolon-separated authentication configs. Default: auth=SIMPLE</description>
+ <description>Semicolon-separated authentication configs.</description>
+ <placeholder>auth=SIMPLE</placeholder>
+ <default-value>auth=SIMPLE</default-value>
<required>false</required>
</parameter>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/hive/pom.xml b/contrib/views/hive/pom.xml
index 0ed6af4..e381719 100644
--- a/contrib/views/hive/pom.xml
+++ b/contrib/views/hive/pom.xml
@@ -195,7 +195,7 @@
<artifactId>frontend-maven-plugin</artifactId>
<version>0.0.14</version>
<configuration>
- <nodeVersion>v0.10.26</nodeVersion>
+ <nodeVersion>v0.10.32</nodeVersion>
<npmVersion>1.4.3</npmVersion>
<workingDirectory>src/main/resources/ui/hive-web/</workingDirectory>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
index e28193d..a963975 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
@@ -20,9 +20,8 @@ package org.apache.ambari.view.hive;
import com.google.inject.Inject;
import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.hive.persistence.Storage;
-import org.apache.ambari.view.hive.persistence.utils.StorageUtil;
import org.apache.ambari.view.hive.utils.HdfsApi;
+import org.apache.ambari.view.hive.utils.SharedObjectsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,19 +36,16 @@ public class BaseService {
protected final static Logger LOG =
LoggerFactory.getLogger(BaseService.class);
- private Storage storage;
- protected Storage getStorage() {
- if (storage == null) {
- storage = StorageUtil.getInstance(context).getStorage();
+ private SharedObjectsFactory sharedObjectsFactory;
+ public SharedObjectsFactory getSharedObjectsFactory() {
+ if (sharedObjectsFactory == null) {
+ sharedObjectsFactory = new SharedObjectsFactory(context);
}
- return storage;
+ return sharedObjectsFactory;
}
- private HdfsApi hdfsApi = null;
- protected HdfsApi getHdfsApi() {
- if (hdfsApi == null)
- hdfsApi = HdfsApi.getInstance(context);
- return hdfsApi;
+ public void setSharedObjectsFactory(SharedObjectsFactory sharedObjectsFactory) {
+ this.sharedObjectsFactory = sharedObjectsFactory;
}
public BaseService() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
index f975393..870e31a 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
@@ -66,7 +66,7 @@ public class HelpService extends BaseService {
public Response testStorage(){
TestBean test = new TestBean();
test.someData = "hello world";
- getStorage().store(TestBean.class, test);
+ getSharedObjectsFactory().getStorage().store(TestBean.class, test);
return Response.ok("OK").build();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
index fce0177..d298931 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
@@ -22,15 +22,15 @@ import org.apache.ambari.view.hive.persistence.utils.Indexed;
public class TestBean implements Indexed {
public String someData;
- public Integer id;
+ public String id;
@Override
- public Integer getId() {
+ public String getId() {
return id;
}
@Override
- public void setId(Integer id) {
+ public void setId(String id) {
this.id = id;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
index e713aba..44c90ae 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
@@ -18,7 +18,9 @@
package org.apache.ambari.view.hive.client;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.service.auth.HiveAuthFactory;
import org.apache.hive.service.auth.KerberosSaslHelper;
import org.apache.hive.service.auth.PlainSaslHelper;
@@ -54,11 +56,13 @@ public class Connection {
private TTransport transport;
private DDLDelegator ddl;
+ private String username;
- public Connection(String host, int port, Map<String, String> authParams) throws HiveClientException {
+ public Connection(String host, int port, Map<String, String> authParams, String username) throws HiveClientException {
this.host = host;
this.port = port;
this.authParams = authParams;
+ this.username = username;
openConnection();
ddl = new DDLDelegator(this);
@@ -78,11 +82,11 @@ public class Connection {
+ host + ":" + port + ": " + e.toString(), e);
}
LOG.info("Hive connection opened");
- openSession();
}
/**
* Based on JDBC implementation of HiveConnection.createBinaryTransport
+ *
* @return transport
* @throws HiveClientException
*/
@@ -107,6 +111,11 @@ public class Connection {
}
saslProps.put(Sasl.QOP, saslQOP.toString());
saslProps.put(Sasl.SERVER_AUTH, "true");
+
+ Configuration conf = new Configuration();
+ conf.set("hadoop.security.authentication", "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+
transport = KerberosSaslHelper.getKerberosTransport(
authParams.get(Utils.HiveAuthenticationParams.AUTH_PRINCIPAL), host,
HiveAuthFactory.getSocketTransport(host, port, 10000), saslProps,
@@ -119,7 +128,7 @@ public class Connection {
host, HiveAuthFactory.getSocketTransport(host, port, 10000), saslProps);
} else {
// we are using PLAIN Sasl connection with user/password
- String userName = getAuthParamDefault(Utils.HiveAuthenticationParams.AUTH_USER, Utils.HiveAuthenticationParams.ANONYMOUS_USER);
+ String userName = getAuthParamDefault(Utils.HiveAuthenticationParams.AUTH_USER, getUsername());
String passwd = getAuthParamDefault(Utils.HiveAuthenticationParams.AUTH_PASSWD, Utils.HiveAuthenticationParams.ANONYMOUS_USER);
// Note: Thrift returns an SSL socket that is already bound to the specified host:port
// Therefore an open called on this would be a no-op later
@@ -250,6 +259,7 @@ public class Connection {
public TExecuteStatementResp body() throws HiveClientException {
TExecuteStatementReq execReq = null;
+ openSession();
execReq = new TExecuteStatementReq(getSessHandle(), oneCmd);
execReq.setRunAsync(async);
execReq.setConfOverlay(new HashMap<String, String>()); //maybe it's hive configuration? use it, Luke!
@@ -398,4 +408,12 @@ public class Connection {
public void setAuthParams(Map<String, String> authParams) {
this.authParams = authParams;
}
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
new file mode 100644
index 0000000..6886f57
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.client;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.utils.ServiceFormattedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ConnectionFactory implements IConnectionFactory {
+ private final static Logger LOG =
+ LoggerFactory.getLogger(ConnectionFactory.class);
+ private ViewContext context;
+
+ public ConnectionFactory(ViewContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Connection getHiveConnection() {
+ try {
+ return new Connection(getHiveHost(), Integer.valueOf(getHivePort()),
+ getHiveAuthParams(), context.getUsername());
+ } catch (HiveClientException e) {
+ throw new ServiceFormattedException("Couldn't open connection to Hive: " + e.toString(), e);
+ }
+ }
+
+ private String getHiveHost() {
+ return context.getProperties().get("hive.host");
+ }
+
+ private String getHivePort() {
+ return context.getProperties().get("hive.port");
+ }
+
+ private Map<String, String> getHiveAuthParams() {
+ String auth = context.getProperties().get("hive.auth");
+ Map<String, String> params = new HashMap<String, String>();
+ if (auth == null || auth.isEmpty()) {
+ auth = "auth=NOSASL";
+ }
+ for(String param : auth.split(";")) {
+ String[] keyvalue = param.split("=");
+ if (keyvalue.length != 2) {
+ LOG.error("Can not parse authentication param " + param + " in " + auth);
+ continue;
+ }
+ params.put(keyvalue[0], keyvalue[1]);
+ }
+ return params;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java
deleted file mode 100644
index 6d07067..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.client;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.hive.utils.ServiceFormattedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class ConnectionPool {
- private final static Logger LOG =
- LoggerFactory.getLogger(ConnectionPool.class);
-
- private static Map<String, Connection> viewSingletonObjects = new HashMap<String, Connection>();
- /**
- * Returns HdfsApi object specific to instance
- * @param context View Context instance
- * @return Hdfs business delegate object
- */
- public static Connection getConnection(ViewContext context) {
- if (!viewSingletonObjects.containsKey(context.getInstanceName()))
- viewSingletonObjects.put(context.getInstanceName(), connectToHive(context));
- return viewSingletonObjects.get(context.getInstanceName());
- }
-
- private static Connection connectToHive(ViewContext context) {
- try {
- return new Connection(getHiveHost(context), Integer.valueOf(getHivePort(context)), getHiveAuthParams(context));
- } catch (HiveClientException e) {
- throw new ServiceFormattedException("Couldn't open connection to Hive: " + e.toString(), e);
- }
- }
-
- public static void setInstance(ViewContext context, Connection api) {
- viewSingletonObjects.put(context.getInstanceName(), api);
- }
-
- private static String getHiveHost(ViewContext context) {
- return context.getProperties().get("hive.host");
- }
-
- private static String getHivePort(ViewContext context) {
- return context.getProperties().get("hive.port");
- }
-
- private static Map<String, String> getHiveAuthParams(ViewContext context) {
- String auth = context.getProperties().get("hive.auth");
- Map<String, String> params = new HashMap<String, String>();
- if (auth == null || auth.isEmpty()) {
- auth = "auth=NOSASL";
- }
- for(String param : auth.split(";")) {
- String[] keyvalue = param.split("=");
- if (keyvalue.length != 2) {
- LOG.error("Can not parse authentication param " + param + " in " + auth);
- continue;
- }
- params.put(keyvalue[0], keyvalue[1]);
- }
- return params;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java
new file mode 100644
index 0000000..7a3df06
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.client;
+
+
+public interface IConnectionFactory {
+ Connection getHiveConnection();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
index 7a4ff40..b4bc415 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
import javax.ws.rs.WebApplicationException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -55,7 +54,7 @@ public class DataStoreStorage implements Storage {
public synchronized void store(Class model, Indexed obj) {
try {
if (obj.getId() == null) {
- int id = nextIdForEntity(context, model);
+ String id = nextIdForEntity(context, model);
obj.setId(id);
}
context.getDataStore().store(obj);
@@ -64,7 +63,7 @@ public class DataStoreStorage implements Storage {
}
}
- private static synchronized int nextIdForEntity(ViewContext context, Class aClass) {
+ private static synchronized String nextIdForEntity(ViewContext context, Class aClass) {
// auto increment id implementation
String lastId = context.getInstanceData(aClass.getName());
int newId;
@@ -74,12 +73,12 @@ public class DataStoreStorage implements Storage {
newId = Integer.parseInt(lastId) + 1;
}
context.putInstanceData(aClass.getName(), String.valueOf(newId));
- return newId;
+ return String.valueOf(newId);
}
@Override
- public synchronized <T extends Indexed> T load(Class<T> model, Integer id) throws ItemNotFound {
- LOG.debug(String.format("Loading %s #%d", model.getName(), id));
+ public synchronized <T extends Indexed> T load(Class<T> model, Object id) throws ItemNotFound {
+ LOG.debug(String.format("Loading %s #%s", model.getName(), id));
try {
T obj = context.getDataStore().find(model, id);
if (obj != null) {
@@ -97,10 +96,9 @@ public class DataStoreStorage implements Storage {
LinkedList<T> list = new LinkedList<T>();
LOG.debug(String.format("Loading all %s-s", model.getName()));
try {
- for(T item: context.getDataStore().findAll(model, null)) {
- if ((filter == null) || filter.isConform(item)) {
- list.add(item);
- }
+ //TODO: use WHERE statement instead of this ugly filter
+ for(T item: context.getDataStore().findAll(model, filter.whereStatement())) {
+ list.add(item);
}
} catch (PersistenceException e) {
throw new ServiceFormattedException("Error while finding all objects in DataStorage", e);
@@ -124,8 +122,8 @@ public class DataStoreStorage implements Storage {
}
@Override
- public synchronized void delete(Class model, int id) throws ItemNotFound {
- LOG.debug(String.format("Deleting %s:%d", model.getName(), id));
+ public synchronized void delete(Class model, Object id) throws ItemNotFound {
+ LOG.debug(String.format("Deleting %s:%s", model.getName(), id));
Object obj = load(model, id);
try {
context.getDataStore().remove(obj);
@@ -135,7 +133,7 @@ public class DataStoreStorage implements Storage {
}
@Override
- public boolean exists(Class model, Integer id) {
+ public boolean exists(Class model, Object id) {
try {
return context.getDataStore().find(model, id) != null;
} catch (PersistenceException e) {
@@ -151,7 +149,7 @@ public class DataStoreStorage implements Storage {
storage.store(SmokeTestEntity.class, entity);
if (entity.getId() == null) throw new ServiceFormattedException("Ambari Views instance data DB doesn't work properly (auto increment id doesn't work)", null);
- Integer id = entity.getId();
+ Object id = entity.getId();
SmokeTestEntity entity2 = storage.load(SmokeTestEntity.class, id);
boolean status = entity2.getData().compareTo("42") == 0;
storage.delete(SmokeTestEntity.class, id);
@@ -164,14 +162,14 @@ public class DataStoreStorage implements Storage {
}
public static class SmokeTestEntity implements Indexed {
- private Integer id = null;
+ private String id = null;
private String data = null;
- public Integer getId() {
+ public String getId() {
return id;
}
- public void setId(Integer id) {
+ public void setId(String id) {
this.id = id;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java
new file mode 100644
index 0000000..298d4c8
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.persistence;
+
+public interface IStorageFactory {
+ Storage getStorage();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
index e8a904c..6e88063 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
@@ -62,7 +62,7 @@ public abstract class KeyValueStorage implements Storage {
int lastIndex = getConfig().getInt(modelIndexingPropName, 0);
lastIndex ++;
getConfig().setProperty(modelIndexingPropName, lastIndex);
- obj.setId(lastIndex);
+ obj.setId(String.valueOf(lastIndex));
}
String modelPropName = getItemPropertyName(model, obj.getId());
@@ -71,12 +71,13 @@ public abstract class KeyValueStorage implements Storage {
}
@Override
- public <T extends Indexed> T load(Class<T> model, Integer id) throws ItemNotFound {
+ public <T extends Indexed> T load(Class<T> model, Object id) throws ItemNotFound {
String modelPropName = getItemPropertyName(model, id);
LOG.debug(String.format("Loading %s", modelPropName));
if (getConfig().containsKey(modelPropName)) {
String json = read(modelPropName);
LOG.debug(String.format("json: %s", json));
+
return deserialize(model, json);
} else {
throw new ItemNotFound();
@@ -141,14 +142,14 @@ public abstract class KeyValueStorage implements Storage {
}
@Override
- public synchronized void delete(Class model, int id) {
- LOG.debug(String.format("Deleting %s:%d", model.getName(), id));
+ public synchronized void delete(Class model, Object id) {
+ LOG.debug(String.format("Deleting %s:%s", model.getName(), id));
String modelPropName = getItemPropertyName(model, id);
clear(modelPropName);
}
@Override
- public boolean exists(Class model, Integer id) {
+ public boolean exists(Class model, Object id) {
return getConfig().containsKey(getItemPropertyName(model, id));
}
@@ -156,7 +157,7 @@ public abstract class KeyValueStorage implements Storage {
return String.format("%s:index", model.getName());
}
- private String getItemPropertyName(Class model, int id) {
- return String.format("%s.%d", model.getName(), id);
+ private String getItemPropertyName(Class model, Object id) {
+ return String.format("%s.%s", model.getName(), id);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
index fb75670..188282e 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
@@ -41,7 +41,7 @@ public interface Storage {
* @return bean instance
* @throws ItemNotFound thrown if item with id was not found in DB
*/
- <T extends Indexed> T load(Class<T> model, Integer id) throws ItemNotFound;
+ <T extends Indexed> T load(Class<T> model, Object id) throws ItemNotFound;
/**
* Load all objects of given bean class
@@ -74,7 +74,7 @@ public interface Storage {
* @param model bean class
* @param id identifier
*/
- void delete(Class model, int id) throws ItemNotFound;
+ void delete(Class model, Object id) throws ItemNotFound;
/**
* Check is object exists
@@ -82,5 +82,5 @@ public interface Storage {
* @param id identifier
* @return true if exists
*/
- boolean exists(Class model, Integer id);
+ boolean exists(Class model, Object id);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
index 79aa2f9..eba572e 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
@@ -28,4 +28,5 @@ public interface FilteringStrategy {
* @return true if item conforms this filter
*/
boolean isConform(Indexed item);
+ String whereStatement();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
index 1d80920..82b7d57 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
@@ -26,11 +26,11 @@ public interface Indexed {
* Get the ID
* @return ID
*/
- Integer getId();
+ String getId();
/**
* Set ID
* @param id ID
*/
- void setId(Integer id);
+ void setId(String id);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
index 9f2b4a2..620f440 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
@@ -30,4 +30,9 @@ public class OnlyOwnersFilteringStrategy implements FilteringStrategy {
Owned object = (Owned) item;
return object.getOwner().compareTo(username) == 0;
}
+
+ @Override
+ public String whereStatement() {
+ return "owner = '" + username + "'";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java
new file mode 100644
index 0000000..88a6d66
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.persistence.utils;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.DataStoreStorage;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
+import org.apache.ambari.view.hive.persistence.LocalKeyValueStorage;
+import org.apache.ambari.view.hive.persistence.Storage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Storage factory, creates storage of Local or Persistence API type.
+ * Type depends on context configuration: if "dataworker.storagePath" is set,
+ * storage of Local type will be created. Otherwise, Persistence API will be used.
+ *
+ * Storage is singleton.
+ */
+public class StorageFactory implements IStorageFactory {
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(StorageFactory.class);
+
+ private ViewContext context;
+
+ /**
+ * Constructor of storage factory
+ * @param context View Context instance
+ */
+ public StorageFactory(ViewContext context) {
+ this.context = context;
+ }
+
+ /**
+ * Creates storage instance
+ * @return storage instance
+ */
+ public Storage getStorage() {
+ String fileName = context.getProperties().get("dataworker.storagePath");
+
+ Storage storageInstance;
+ if (fileName != null) {
+ LOG.debug("Using local storage in " + fileName + " to store data");
+ // If specifed, use LocalKeyValueStorage - key-value file based storage
+ storageInstance = new LocalKeyValueStorage(context);
+ } else {
+ LOG.debug("Using Persistence API to store data");
+ // If not specifed, use ambari-views Persistence API
+ storageInstance = new DataStoreStorage(context);
+ }
+ return storageInstance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java
deleted file mode 100644
index 1e67c02..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.persistence.utils;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.hive.persistence.DataStoreStorage;
-import org.apache.ambari.view.hive.persistence.LocalKeyValueStorage;
-import org.apache.ambari.view.hive.persistence.Storage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Storage factory, creates storage of Local or Persistence API type.
- * Type depends on context configuration: if "dataworker.storagePath" is set,
- * storage of Local type will be created. Otherwise, Persistence API will be used.
- *
- * Storage is singleton.
- */
-public class StorageUtil {
- private Storage storageInstance = null;
-
- protected final static Logger LOG =
- LoggerFactory.getLogger(StorageUtil.class);
-
-
- private static Map<String, StorageUtil> viewSingletonObjects = new HashMap<String, StorageUtil>();
- public static StorageUtil getInstance(ViewContext context) {
- if (!viewSingletonObjects.containsKey(context.getInstanceName()))
- viewSingletonObjects.put(context.getInstanceName(), new StorageUtil(context));
- return viewSingletonObjects.get(context.getInstanceName());
- }
-
- public static void dropAllConnections() {
- viewSingletonObjects.clear();
- }
-
- private ViewContext context;
-
- /**
- * Constructor of storage util
- * @param context View Context instance
- */
- public StorageUtil(ViewContext context) {
- this.context = context;
- }
-
- /**
- * Get storage instance. If one is not created, creates instance.
- * @return storage instance
- */
- public synchronized Storage getStorage() {
- if (storageInstance == null) {
- String fileName = context.getProperties().get("dataworker.storagePath");
- if (fileName != null) {
- LOG.debug("Using local storage in " + fileName + " to store data");
- // If specifed, use LocalKeyValueStorage - key-value file based storage
- storageInstance = new LocalKeyValueStorage(context);
- } else {
- LOG.debug("Using Persistence API to store data");
- // If not specifed, use ambari-views Persistence API
- storageInstance = new DataStoreStorage(context);
- }
- }
- return storageInstance;
- }
-
- /**
- * Set storage to use across all application.
- * Used in unit tests.
- * @param storage storage instance
- */
- public void setStorage(Storage storage) {
- storageInstance = storage;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
index b8dec17..28cf4d1 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
@@ -18,12 +18,11 @@
package org.apache.ambari.view.hive.resources;
-import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
import org.apache.ambari.view.hive.persistence.Storage;
import org.apache.ambari.view.hive.persistence.utils.FilteringStrategy;
import org.apache.ambari.view.hive.persistence.utils.Indexed;
import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.hive.persistence.utils.StorageUtil;
import org.apache.ambari.view.hive.utils.ServiceFormattedException;
import java.util.List;
@@ -32,18 +31,20 @@ import java.util.List;
* CRUD resource manager
* @param <T> Data type with ID
*/
-abstract public class CRUDResourceManager<T extends Indexed> {
+abstract public class CRUDResourceManager<T extends Indexed> implements IResourceManager<T> {
//TODO: refactor: generic parameter gets Fabric for Indexed objects, not objects itself
private Storage storage = null;
protected final Class<? extends T> resourceClass;
+ protected IStorageFactory storageFabric;
/**
* Constructor
* @param resourceClass model class
*/
- public CRUDResourceManager(Class<? extends T> resourceClass) {
+ public CRUDResourceManager(Class<? extends T> resourceClass, IStorageFactory storageFabric) {
this.resourceClass = resourceClass;
+ this.storageFabric = storageFabric;
}
// CRUD operations
@@ -52,7 +53,8 @@ abstract public class CRUDResourceManager<T extends Indexed> {
* @param object object
* @return model object
*/
- protected T create(T object) {
+ @Override
+ public T create(T object) {
object.setId(null);
return this.save(object);
}
@@ -63,9 +65,10 @@ abstract public class CRUDResourceManager<T extends Indexed> {
* @return model object
* @throws org.apache.ambari.view.hive.persistence.utils.ItemNotFound
*/
- protected T read(Integer id) throws ItemNotFound {
+ @Override
+ public T read(Object id) throws ItemNotFound {
T object = null;
- object = getStorage().load(this.resourceClass, id);
+ object = storageFabric.getStorage().load(this.resourceClass, id);
if (!checkPermissions(object))
throw new ItemNotFound();
return object;
@@ -76,8 +79,9 @@ abstract public class CRUDResourceManager<T extends Indexed> {
* @param filteringStrategy filtering strategy
* @return list of filtered objects
*/
- protected List<T> readAll(FilteringStrategy filteringStrategy) {
- return getStorage().loadAll(this.resourceClass, filteringStrategy);
+ @Override
+ public List<T> readAll(FilteringStrategy filteringStrategy) {
+ return storageFabric.getStorage().loadAll(this.resourceClass, filteringStrategy);
}
/**
@@ -87,7 +91,8 @@ abstract public class CRUDResourceManager<T extends Indexed> {
* @return model object
* @throws org.apache.ambari.view.hive.persistence.utils.ItemNotFound
*/
- protected T update(T newObject, Integer id) throws ItemNotFound {
+ @Override
+ public T update(T newObject, String id) throws ItemNotFound {
newObject.setId(id);
this.save(newObject);
return newObject;
@@ -98,29 +103,22 @@ abstract public class CRUDResourceManager<T extends Indexed> {
* @param resourceId object identifier
* @throws org.apache.ambari.view.hive.persistence.utils.ItemNotFound
*/
- protected void delete(Integer resourceId) throws ItemNotFound {
- if (!getStorage().exists(this.resourceClass, resourceId)) {
+ @Override
+ public void delete(Object resourceId) throws ItemNotFound {
+ if (!storageFabric.getStorage().exists(this.resourceClass, resourceId)) {
throw new ItemNotFound();
}
- getStorage().delete(this.resourceClass, resourceId);
+ storageFabric.getStorage().delete(this.resourceClass, resourceId);
}
// UTILS
protected T save(T object) {
- getStorage().store(resourceClass, object);
+ storageFabric.getStorage().store(resourceClass, object);
return object;
}
- protected Storage getStorage() {
- if (storage == null) {
- storage = StorageUtil.getInstance(getContext()).getStorage();
- }
- return storage;
- }
-
protected abstract boolean checkPermissions(T object);
- protected abstract ViewContext getContext();
protected void cleanupAfterErrorAndThrowAgain(Indexed object, ServiceFormattedException e) {
try {
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java
new file mode 100644
index 0000000..222d695
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources;
+
+import org.apache.ambari.view.hive.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.hive.persistence.utils.Indexed;
+import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
+
+import java.util.List;
+
+public interface IResourceManager<T extends Indexed> {
+ T create(T object);
+
+ T read(Object id) throws ItemNotFound;
+
+ List<T> readAll(FilteringStrategy filteringStrategy);
+
+ T update(T newObject, String id) throws ItemNotFound;
+
+ void delete(Object resourceId) throws ItemNotFound;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
index d1223fb..15fcf22 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
@@ -19,6 +19,7 @@
package org.apache.ambari.view.hive.resources;
import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
import org.apache.ambari.view.hive.persistence.utils.PersonalResource;
import org.slf4j.Logger;
@@ -31,24 +32,24 @@ import java.util.concurrent.Callable;
* @param <T> Data type with ID and Owner
*/
public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRUDResourceManager<T> {
- protected ViewContext context;
protected boolean ignorePermissions = false;
private final static Logger LOG =
LoggerFactory.getLogger(PersonalCRUDResourceManager.class);
+ protected ViewContext context;
+
/**
* Constructor
* @param resourceClass model class
- * @param context View Context instance
*/
- public PersonalCRUDResourceManager(Class<? extends T> resourceClass, ViewContext context) {
- super(resourceClass);
+ public PersonalCRUDResourceManager(Class<? extends T> resourceClass, IStorageFactory storageFabric, ViewContext context) {
+ super(resourceClass, storageFabric);
this.context = context;
}
@Override
- public T update(T newObject, Integer id) throws ItemNotFound {
- T object = getStorage().load(this.resourceClass, id);
+ public T update(T newObject, String id) throws ItemNotFound {
+ T object = storageFabric.getStorage().load(this.resourceClass, id);
if (object.getOwner().compareTo(this.context.getUsername()) != 0) {
throw new ItemNotFound();
}
@@ -74,18 +75,13 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU
return object.getOwner().compareTo(this.context.getUsername()) == 0;
}
- @Override
- public ViewContext getContext() {
- return context;
- }
-
/**
* Execute action ignoring objects owner
* @param actions callable to execute
* @return value returned from actions
* @throws Exception
*/
- public <T> T ignorePermissions(Callable<T> actions) throws Exception {
+ public T ignorePermissions(Callable<T> actions) throws Exception {
ignorePermissions = true;
T result;
try {
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
index 0d2b297..9c4ca36 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
@@ -19,6 +19,7 @@
package org.apache.ambari.view.hive.resources;
import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
import org.apache.ambari.view.hive.persistence.utils.Indexed;
/**
@@ -31,20 +32,13 @@ public class SharedCRUDResourceManager<T extends Indexed> extends CRUDResourceMa
/**
* Constructor
* @param responseClass model class
- * @param context View Context instance
*/
- public SharedCRUDResourceManager(Class<T> responseClass, ViewContext context) {
- super(responseClass);
- this.context = context;
+ public SharedCRUDResourceManager(Class<T> responseClass, IStorageFactory storageFabric) {
+ super(responseClass, storageFabric);
}
@Override
protected boolean checkPermissions(T object) {
return true; //everyone has permission
}
-
- @Override
- protected ViewContext getContext() {
- return context;
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
index e5983b9..bb1a0a2 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
@@ -21,13 +21,13 @@ package org.apache.ambari.view.hive.resources.browser;
import com.google.inject.Inject;
import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.ViewResourceHandler;
-import org.apache.ambari.view.hive.BaseService;
import org.apache.ambari.view.hive.client.ColumnDescription;
-import org.apache.ambari.view.hive.client.ConnectionPool;
import org.apache.ambari.view.hive.client.Cursor;
+import org.apache.ambari.view.hive.client.IConnectionFactory;
import org.apache.ambari.view.hive.resources.jobs.ResultsPaginationController;
import org.apache.ambari.view.hive.utils.BadRequestFormattedException;
import org.apache.ambari.view.hive.utils.ServiceFormattedException;
+import org.apache.ambari.view.hive.utils.SharedObjectsFactory;
import org.apache.commons.collections4.map.PassiveExpiringMap;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
@@ -55,6 +55,8 @@ public class HiveBrowserService {
private static final long EXPIRING_TIME = 10*60*1000; // 10 minutes
private static Map<String, Cursor> resultsCache;
+ private IConnectionFactory connectionFactory;
+
public static Map<String, Cursor> getResultsCache() {
if (resultsCache == null) {
PassiveExpiringMap<String, Cursor> resultsCacheExpiringMap =
@@ -64,6 +66,12 @@ public class HiveBrowserService {
return resultsCache;
}
+ private IConnectionFactory getConnectionFactory() {
+ if (connectionFactory == null)
+ connectionFactory = new SharedObjectsFactory(context);
+ return new SharedObjectsFactory(context);
+ }
+
/**
* Returns list of databases
*/
@@ -81,7 +89,7 @@ public class HiveBrowserService {
String curl = null;
try {
JSONObject response = new JSONObject();
- List<String> tables = ConnectionPool.getConnection(context).ddl().getDBList(like);
+ List<String> tables = getConnectionFactory().getHiveConnection().ddl().getDBList(like);
response.put("databases", tables);
return Response.ok(response).build();
} catch (WebApplicationException ex) {
@@ -116,7 +124,7 @@ public class HiveBrowserService {
new Callable<Cursor>() {
@Override
public Cursor call() throws Exception {
- return ConnectionPool.getConnection(context).ddl().getDBListCursor(finalLike);
+ return getConnectionFactory().getHiveConnection().ddl().getDBListCursor(finalLike);
}
}).build();
} catch (WebApplicationException ex) {
@@ -146,7 +154,7 @@ public class HiveBrowserService {
String curl = null;
try {
JSONObject response = new JSONObject();
- List<String> tables = ConnectionPool.getConnection(context).ddl().getTableList(db, like);
+ List<String> tables = getConnectionFactory().getHiveConnection().ddl().getTableList(db, like);
response.put("tables", tables);
response.put("database", db);
return Response.ok(response).build();
@@ -183,7 +191,7 @@ public class HiveBrowserService {
new Callable<Cursor>() {
@Override
public Cursor call() throws Exception {
- Cursor cursor = ConnectionPool.getConnection(context).ddl().getTableListCursor(db, finalLike);
+ Cursor cursor = getConnectionFactory().getHiveConnection().ddl().getTableListCursor(db, finalLike);
cursor.selectColumns(requestedColumns);
return cursor;
}
@@ -212,7 +220,7 @@ public class HiveBrowserService {
String curl = null;
try {
JSONObject response = new JSONObject();
- List<ColumnDescription> columnDescriptions = ConnectionPool.getConnection(context).ddl()
+ List<ColumnDescription> columnDescriptions = getConnectionFactory().getHiveConnection().ddl()
.getTableDescription(db, table, like, extendedTableDescription);
response.put("columns", columnDescriptions);
response.put("database", db);
@@ -247,7 +255,7 @@ public class HiveBrowserService {
new Callable<Cursor>() {
@Override
public Cursor call() throws Exception {
- Cursor cursor = ConnectionPool.getConnection(context).ddl().getTableDescriptionCursor(db, table, like);
+ Cursor cursor = getConnectionFactory().getHiveConnection().ddl().getTableDescriptionCursor(db, table, like);
cursor.selectColumns(requestedColumns);
return cursor;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
index 8d886d5..3f5b3b8 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
@@ -23,6 +23,7 @@ import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.ViewResourceHandler;
import org.apache.ambari.view.hive.BaseService;
import org.apache.ambari.view.hive.utils.*;
+import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.json.simple.JSONObject;
@@ -51,6 +52,8 @@ import java.io.IOException;
* update file content
*/
public class FileService extends BaseService {
+ public static final String FAKE_FILE = "fakefile://";
+
@Inject
ViewResourceHandler handler;
@@ -66,17 +69,23 @@ public class FileService extends BaseService {
public Response getFilePage(@PathParam("filePath") String filePath, @QueryParam("page") Long page) throws IOException, InterruptedException {
LOG.debug("Reading file " + filePath);
try {
- FilePaginator paginator = new FilePaginator(filePath, context);
+ FileResource file = new FileResource();
if (page == null)
page = 0L;
- FileResource file = new FileResource();
- file.setFilePath(filePath);
- file.setFileContent(paginator.readPage(page));
- file.setHasNext(paginator.pageCount() > page + 1);
- file.setPage(page);
- file.setPageCount(paginator.pageCount());
+ if (filePath.startsWith(FAKE_FILE)) {
+ if (page > 1)
+ throw new IllegalArgumentException("There's only one page in fake files");
+
+ String content = filePath.substring(FAKE_FILE.length());
+
+ fillFakeFileObject(filePath, file, content);
+ } else {
+ FilePaginator paginator = new FilePaginator(filePath, getSharedObjectsFactory().getHdfsApi());
+
+ fillRealFileObject(filePath, page, file, paginator);
+ }
JSONObject object = new JSONObject();
object.put("file", file);
@@ -92,6 +101,24 @@ public class FileService extends BaseService {
}
}
+ public void fillRealFileObject(String filePath, Long page, FileResource file, FilePaginator paginator) throws IOException, InterruptedException {
+ file.setFilePath(filePath);
+ file.setFileContent(paginator.readPage(page));
+ file.setHasNext(paginator.pageCount() > page + 1);
+ file.setPage(page);
+ file.setPageCount(paginator.pageCount());
+ }
+
+ public void fillFakeFileObject(String filePath, FileResource file, String encodedContent) {
+ String content = new String(Base64.decodeBase64(encodedContent));
+
+ file.setFilePath(filePath);
+ file.setFileContent(content);
+ file.setHasNext(false);
+ file.setPage(0);
+ file.setPageCount(1);
+ }
+
/**
* Delete single item
*/
@@ -100,7 +127,7 @@ public class FileService extends BaseService {
public Response deleteFile(@PathParam("filePath") String filePath) throws IOException, InterruptedException {
try {
LOG.debug("Deleting file " + filePath);
- if (getHdfsApi().delete(filePath, false)) {
+ if (getSharedObjectsFactory().getHdfsApi().delete(filePath, false)) {
return Response.status(204).build();
}
throw new NotFoundFormattedException("FileSystem.delete returned false", null);
@@ -121,7 +148,7 @@ public class FileService extends BaseService {
@PathParam("filePath") String filePath) throws IOException, InterruptedException {
try {
LOG.debug("Rewriting file " + filePath);
- FSDataOutputStream output = getHdfsApi().create(filePath, true);
+ FSDataOutputStream output = getSharedObjectsFactory().getHdfsApi().create(filePath, true);
output.writeBytes(request.file.getFileContent());
output.close();
return Response.status(204).build();
@@ -143,7 +170,7 @@ public class FileService extends BaseService {
try {
LOG.debug("Creating file " + request.file.getFilePath());
try {
- FSDataOutputStream output = getHdfsApi().create(request.file.getFilePath(), false);
+ FSDataOutputStream output = getSharedObjectsFactory().getHdfsApi().create(request.file.getFilePath(), false);
if (request.file.getFileContent() != null) {
output.writeBytes(request.file.getFileContent());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
new file mode 100644
index 0000000..ce82e15
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.jobs;
+
+import org.apache.ambari.view.hive.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.hive.persistence.utils.Indexed;
+import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.hive.persistence.utils.OnlyOwnersFilteringStrategy;
+import org.apache.ambari.view.hive.resources.IResourceManager;
+import org.apache.ambari.view.hive.resources.jobs.atsJobs.HiveQueryId;
+import org.apache.ambari.view.hive.resources.jobs.atsJobs.IATSParser;
+import org.apache.ambari.view.hive.resources.jobs.atsJobs.TezDagId;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.Job;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl;
+import org.apache.commons.beanutils.PropertyUtils;
+import org.apache.commons.codec.binary.Base64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * View Jobs and ATS Jobs aggregator
+ * Not all ViewJobs create ATS job
+ */
+public class Aggregator {
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(Aggregator.class);
+
+ private final IATSParser ats;
+ private final IOperationHandleResourceManager operationHandleResourceManager;
+ private IResourceManager<Job> viewJobResourceManager;
+
+ public Aggregator(IResourceManager<Job> jobResourceManager,
+ IOperationHandleResourceManager operationHandleResourceManager,
+ IATSParser ats) {
+ this.viewJobResourceManager = jobResourceManager;
+ this.operationHandleResourceManager = operationHandleResourceManager;
+ this.ats = ats;
+ }
+
+ public List<Job> readAll(String username) {
+ Set<String> addedOperationIds = new HashSet<String>();
+
+ List<Job> allJobs = new LinkedList<Job>();
+ for (HiveQueryId atsHiveQuery : ats.getHiveQuieryIdsList(username)) {
+
+ TezDagId atsTezDag;
+ if (atsHiveQuery.dagNames != null && atsHiveQuery.dagNames.size() > 0) {
+ String dagName = atsHiveQuery.dagNames.get(0);
+
+ atsTezDag = ats.getTezDAGByName(dagName);
+ } else {
+ atsTezDag = new TezDagId();
+ }
+
+ JobImpl atsJob;
+ if (hasOperationId(atsHiveQuery)) {
+ try {
+ Job viewJob = getJobByOperationId(urlSafeBase64ToHexString(atsHiveQuery.operationId));
+ saveJobInfoIfNeeded(atsHiveQuery, atsTezDag, viewJob);
+
+ atsJob = mergeAtsJobWithViewJob(atsHiveQuery, atsTezDag, viewJob);
+ } catch (ItemNotFound itemNotFound) {
+ // Executed from HS2, but outside of Hive View
+ atsJob = atsOnlyJob(atsHiveQuery, atsTezDag);
+ }
+ } else {
+ atsJob = atsOnlyJob(atsHiveQuery, atsTezDag);
+ }
+ allJobs.add(atsJob);
+
+ addedOperationIds.add(atsHiveQuery.operationId);
+ }
+
+ //cover case when operationId is present, but not exists in ATS
+ //e.g. optimized queries without executing jobs, like "SELECT * FROM TABLE"
+ for (Job job : viewJobResourceManager.readAll(new OnlyOwnersFilteringStrategy(username))) {
+ List<StoredOperationHandle> operationHandles = operationHandleResourceManager.readJobRelatedHandles(job);
+ assert operationHandles.size() <= 1;
+
+ if (operationHandles.size() > 0) {
+ StoredOperationHandle operationHandle = operationHandles.get(0);
+
+ if (!addedOperationIds.contains(hexStringToUrlSafeBase64(operationHandle.getGuid()))) {
+ //e.g. query without hadoop job: select * from table
+ allJobs.add(job);
+ }
+ }
+ }
+
+ return allJobs;
+ }
+
+ protected boolean hasOperationId(HiveQueryId atsHiveQuery) {
+ return atsHiveQuery.operationId != null;
+ }
+
+ protected JobImpl mergeAtsJobWithViewJob(HiveQueryId atsHiveQuery, TezDagId atsTezDag, Job viewJob) {
+ JobImpl atsJob;
+ try {
+ atsJob = new JobImpl(PropertyUtils.describe(viewJob));
+ }catch(IllegalAccessException e){
+ LOG.error("Can't instantiate JobImpl", e);
+ return null;
+ }catch(InvocationTargetException e){
+ LOG.error("Can't instantiate JobImpl", e);
+ return null;
+ }catch(NoSuchMethodException e){
+ LOG.error("Can't instantiate JobImpl", e);
+ return null;
+ }
+ fillAtsJobFields(atsJob, atsHiveQuery, atsTezDag);
+ return atsJob;
+ }
+
+ protected void saveJobInfoIfNeeded(HiveQueryId hiveQueryId, TezDagId tezDagId, Job viewJob) throws ItemNotFound {
+ if (viewJob.getDagName() == null) {
+ viewJob.setDagName(tezDagId.dagName);
+ viewJobResourceManager.update(viewJob, viewJob.getId());
+ }
+ if (viewJob.getStatus().equals(tezDagId.status)) {
+ viewJob.setStatus(tezDagId.status);
+ viewJobResourceManager.update(viewJob, viewJob.getId());
+ }
+ }
+
+ protected JobImpl atsOnlyJob(HiveQueryId atsHiveQuery, TezDagId atsTezDag) {
+ JobImpl atsJob = new JobImpl();
+ atsJob.setId(atsHiveQuery.entity);
+ fillAtsJobFields(atsJob, atsHiveQuery, atsTezDag);
+
+ String query = atsHiveQuery.query;
+ atsJob.setTitle(query.substring(0, (query.length() > 42)?42:query.length()));
+
+ atsJob.setQueryFile("fakefile://" + Base64.encodeBase64URLSafeString(query.getBytes())); // fake queryFile
+ return atsJob;
+ }
+
+ protected JobImpl fillAtsJobFields(JobImpl atsJob, HiveQueryId atsHiveQuery, TezDagId atsTezDag) {
+ atsJob.setApplicationId(atsTezDag.applicationId);
+
+ atsJob.setDagName(atsTezDag.dagName);
+ if (!atsTezDag.status.equals(TezDagId.STATUS_UNKNOWN))
+ atsJob.setStatus(atsTezDag.status);
+ if (atsHiveQuery.starttime != 0)
+ atsJob.setDateSubmitted(atsHiveQuery.starttime);
+ atsJob.setDuration(atsHiveQuery.duration);
+ return atsJob;
+ }
+
+ protected Job getJobByOperationId(final String opId) throws ItemNotFound {
+ List<StoredOperationHandle> operationHandles = operationHandleResourceManager.readAll(new FilteringStrategy() {
+ @Override
+ public boolean isConform(Indexed item) {
+ StoredOperationHandle opHandle = (StoredOperationHandle) item;
+ return opHandle.getGuid().equals(opId);
+ }
+
+ @Override
+ public String whereStatement() {
+ return "guid='" + opId + "'";
+ }
+ });
+
+ if (operationHandles.size() != 1)
+ throw new ItemNotFound();
+
+ return viewJobResourceManager.read(operationHandles.get(0).getJobId());
+ }
+
+ protected static String urlSafeBase64ToHexString(String urlsafeBase64){
+ byte[] decoded = Base64.decodeBase64(urlsafeBase64);
+
+ StringBuilder sb = new StringBuilder();
+ for(byte b : decoded){
+ sb.append(String.format("%02x", b));
+ }
+ return sb.toString();
+ }
+
+ protected static String hexStringToUrlSafeBase64(String hexString){
+ byte[] decoded = new byte[hexString.length() / 2];
+
+ for(int i=0; i<hexString.length(); i+=2) {
+ decoded[i / 2] = (byte) Integer.parseInt(String.format("%c%c", hexString.charAt(i), hexString.charAt(i+1)), 16);
+ }
+ return Base64.encodeBase64URLSafeString(decoded);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
index 8f143e7..b70abe2 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
@@ -18,32 +18,19 @@
package org.apache.ambari.view.hive.resources.jobs;
-import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.hive.client.Connection;
-import org.apache.ambari.view.hive.client.ConnectionPool;
import org.apache.ambari.view.hive.client.HiveClientException;
import org.apache.ambari.view.hive.utils.ServiceFormattedException;
import org.apache.hive.service.cli.thrift.TOperationHandle;
-import java.util.HashMap;
-import java.util.Map;
public class ConnectionController {
- private ViewContext context;
- private Connection connection;
private OperationHandleControllerFactory operationHandleControllerFactory;
+ private Connection connection;
- private ConnectionController(ViewContext context) {
- this.context = context;
- connection = ConnectionPool.getConnection(context);
- operationHandleControllerFactory = OperationHandleControllerFactory.getInstance(context);
- }
-
- private static Map<String, ConnectionController> viewSingletonObjects = new HashMap<String, ConnectionController>();
- public static ConnectionController getInstance(ViewContext context) {
- if (!viewSingletonObjects.containsKey(context.getInstanceName()))
- viewSingletonObjects.put(context.getInstanceName(), new ConnectionController(context));
- return viewSingletonObjects.get(context.getInstanceName());
+ public ConnectionController(OperationHandleControllerFactory operationHandleControllerFactory, Connection connection) {
+ this.connection = connection;
+ this.operationHandleControllerFactory = operationHandleControllerFactory;
}
public void selectDatabase(String database) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java
new file mode 100644
index 0000000..185402e
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.jobs;
+
+import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.hive.resources.IResourceManager;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.Job;
+import org.apache.hive.service.cli.thrift.TOperationHandle;
+
+import java.util.List;
+
+public interface IOperationHandleResourceManager extends IResourceManager<StoredOperationHandle> {
+ List<StoredOperationHandle> readJobRelatedHandles(Job job);
+
+ void putHandleForJob(TOperationHandle h, Job job);
+
+ boolean containsHandleForJob(Job job);
+
+ TOperationHandle getHandleForJob(Job job) throws ItemNotFound;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java
deleted file mode 100644
index e6f7a63..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.resources.jobs;
-
-import org.apache.ambari.view.hive.persistence.utils.Indexed;
-import org.apache.ambari.view.hive.persistence.utils.PersonalResource;
-
-import java.io.Serializable;
-
-/**
- * Interface for Job bean to create Proxy for it
- */
-public interface Job extends Serializable,Indexed,PersonalResource {
- public static final String JOB_STATE_UNKNOWN = "Unknown";
- public static final String JOB_STATE_INITIALIZED = "Initialized";
- public static final String JOB_STATE_RUNNING = "Running";
- public static final String JOB_STATE_FINISHED = "Finished";
- public static final String JOB_STATE_CANCELED = "Canceled";
- public static final String JOB_STATE_CLOSED = "Closed";
- public static final String JOB_STATE_ERROR = "Error";
- public static final String JOB_STATE_PENDING = "Pending";
-
- Integer getId();
-
- void setId(Integer id);
-
- String getOwner();
-
- void setOwner(String owner);
-
- String getTitle();
-
- void setTitle(String title);
-
- String getQueryFile();
-
- void setQueryFile(String queryFile);
-
- Long getDateSubmitted();
-
- void setDateSubmitted(Long dateSubmitted);
-
- Long getDuration();
-
- void setDuration(Long duration);
-
- String getStatus();
-
- void setStatus(String status);
-
- String getForcedContent();
-
- void setForcedContent(String forcedContent);
-
- Integer getQueryId();
-
- void setQueryId(Integer queryId);
-
- String getStatusDir();
-
- void setStatusDir(String statusDir);
-
- String getDataBase();
-
- void setDataBase(String dataBase);
-
- String getLogFile();
-
- void setLogFile(String logFile);
-
- String getConfFile();
-
- void setConfFile(String confFile);
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java
deleted file mode 100644
index 3d5189e..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.resources.jobs;
-
-import org.apache.ambari.view.hive.client.Cursor;
-import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
-
-public interface JobController {
- void submit();
-
- void cancel() throws ItemNotFound;
-
- Job getJob();
-
- /**
- * Use carefully. Returns unproxied bean object
- * @return unproxied bean object
- */
- Job getJobPOJO();
-
- Cursor getResults() throws ItemNotFound;
-
- void afterCreation();
-
- void onRead();
-
- boolean isModified();
-
- void clearModified();
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/db973127/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java
deleted file mode 100644
index f6ec5b1..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.resources.jobs;
-
-import org.apache.ambari.view.ViewContext;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class JobControllerFactory {
- private ViewContext context;
-
- private JobControllerFactory(ViewContext context) {
- this.context = context;
- }
-
- private static Map<String, JobControllerFactory> viewSingletonObjects = new HashMap<String, JobControllerFactory>();
- public static JobControllerFactory getInstance(ViewContext context) {
- if (!viewSingletonObjects.containsKey(context.getInstanceName()))
- viewSingletonObjects.put(context.getInstanceName(), new JobControllerFactory(context));
- return viewSingletonObjects.get(context.getInstanceName());
- }
-
- public JobController createControllerForJob(Job job) {
- return new JobControllerImpl(context, job);
- }
-}