You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/03/19 19:33:28 UTC
ambari git commit: Revert "AMBARI-10132. Hive View: Maintain one
session per tab (alexantonenko)"
Repository: ambari
Updated Branches:
refs/heads/branch-2.0.0 8ba5ef274 -> 9c29dfa40
Revert "AMBARI-10132. Hive View: Maintain one session per tab (alexantonenko)"
This reverts commit 8ba5ef2745588e1ce17bd382f93f38a399ccb62e.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9c29dfa4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9c29dfa4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9c29dfa4
Branch: refs/heads/branch-2.0.0
Commit: 9c29dfa40fe97a626f93c50cae2893266713c275
Parents: 8ba5ef2
Author: Alex Antonenko <hi...@gmail.com>
Authored: Thu Mar 19 20:28:52 2015 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Thu Mar 19 20:28:52 2015 +0200
----------------------------------------------------------------------
.../ambari/view/hive/client/Connection.java | 106 ++++++-------------
.../ambari/view/hive/client/DDLDelegator.java | 26 ++---
.../hive/resources/CRUDResourceManager.java | 16 +--
.../resources/PersonalCRUDResourceManager.java | 2 +-
.../resources/browser/HiveBrowserService.java | 20 ++--
.../view/hive/resources/jobs/Aggregator.java | 59 ++---------
.../resources/jobs/ConnectionController.java | 35 +-----
.../view/hive/resources/jobs/JobService.java | 35 ++----
.../jobs/OperationHandleResourceManager.java | 2 +-
.../hive/resources/jobs/atsJobs/ATSParser.java | 10 +-
.../hive/resources/jobs/atsJobs/IATSParser.java | 4 +-
.../hive/resources/jobs/atsJobs/TezDagId.java | 2 +-
.../view/hive/resources/jobs/viewJobs/Job.java | 10 +-
.../jobs/viewJobs/JobControllerImpl.java | 23 ++--
.../hive/resources/jobs/viewJobs/JobImpl.java | 27 +----
.../jobs/viewJobs/JobResourceManager.java | 2 +-
.../savedQueries/SavedQueryResourceManager.java | 7 +-
.../ui/hive-web/app/controllers/index.js | 5 -
.../ui/hive-web/app/controllers/settings.js | 27 ++---
.../ui/hive-web/app/controllers/tez-ui.js | 84 ---------------
.../ui/hive-web/app/initializers/i18n.js | 6 --
.../resources/ui/hive-web/app/models/job.js | 3 -
.../resources/ui/hive-web/app/styles/app.scss | 6 --
.../ui/hive-web/app/templates/index.hbs | 2 +-
.../ui/hive-web/app/templates/tez-ui.hbs | 9 +-
.../view/hive/resources/jobs/ATSParserTest.java | 5 +-
.../hive/resources/jobs/AggregatorTest.java | 10 +-
.../hive/resources/jobs/JobServiceTest.java | 15 +--
28 files changed, 129 insertions(+), 429 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
index 70b5047..44c90ae 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
@@ -18,7 +18,6 @@
package org.apache.ambari.view.hive.client;
-import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.security.UserGroupInformation;
@@ -38,7 +37,6 @@ import javax.security.sasl.Sasl;
import javax.security.sasl.SaslException;
import java.io.IOException;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -53,7 +51,7 @@ public class Connection {
private Map<String, String> authParams;
private TCLIService.Client client = null;
- private Map<String, TSessionHandle> sessHandles = null;
+ private TSessionHandle sessHandle = null;
private TProtocolVersion protocol = null;
private TTransport transport;
@@ -66,8 +64,6 @@ public class Connection {
this.authParams = authParams;
this.username = username;
- this.sessHandles = new HashMap<String, TSessionHandle>();
-
openConnection();
ddl = new DDLDelegator(this);
}
@@ -193,11 +189,8 @@ public class Connection {
return defaultValue;
}
- public synchronized TSessionHandle openSession() throws HiveClientException {
- return openSession(null);
- }
-
- public synchronized TSessionHandle openSession(String forcedTag) throws HiveClientException {
+ private synchronized void openSession() throws HiveClientException {
+ //It's possible to set proxy user configuration here
TOpenSessionResp openResp = new HiveCall<TOpenSessionResp>(this) {
@Override
public TOpenSessionResp body() throws HiveClientException {
@@ -212,41 +205,15 @@ public class Connection {
}.call();
Utils.verifySuccess(openResp.getStatus(), "Unable to open Hive session");
- if (protocol == null)
- protocol = openResp.getServerProtocolVersion();
+ protocol = openResp.getServerProtocolVersion();
+ sessHandle = openResp.getSessionHandle();
LOG.info("Hive session opened");
-
- TSessionHandle sessionHandle = openResp.getSessionHandle();
- String tag;
- if (forcedTag == null)
- tag = Hex.encodeHexString(sessionHandle.getSessionId().getGuid());
- else
- tag = forcedTag;
-
- sessHandles.put(tag, sessionHandle);
-
- return sessionHandle;
}
- public TSessionHandle getSessionByTag(String tag) throws HiveClientException {
- TSessionHandle sessionHandle = sessHandles.get(tag);
- if (sessionHandle == null) {
- throw new HiveClientException("Session with provided tag not found", null);
- }
- return sessionHandle;
- }
-
- public TSessionHandle getOrCreateSessionByTag(String tag) throws HiveClientException {
- try {
- return getSessionByTag(tag);
- } catch (HiveClientException e) {
- return openSession(tag);
- }
- }
-
- private synchronized void closeSession(TSessionHandle sessHandle) throws HiveClientException {
+ private synchronized void closeSession() throws HiveClientException {
if (sessHandle == null) return;
TCloseSessionReq closeReq = new TCloseSessionReq(sessHandle);
+ //It's possible to set proxy user configuration here
TCloseSessionResp closeResp = null;
try {
closeResp = client.CloseSession(closeReq);
@@ -254,28 +221,23 @@ public class Connection {
} catch (TException e) {
throw new HiveClientException("Unable to close Hive session", e);
}
+
+ sessHandle = null;
+ protocol = null;
LOG.info("Hive session closed");
}
public synchronized void closeConnection() throws HiveClientException {
if (client == null) return;
try {
-
- for(Iterator<Map.Entry<String, TSessionHandle>> it = sessHandles.entrySet().iterator(); it.hasNext(); ) {
- Map.Entry<String, TSessionHandle> entry = it.next();
- try {
- closeSession(entry.getValue());
- } catch (HiveClientException e) {
- LOG.error("Unable to close Hive session: " + e.getMessage());
- } finally {
- it.remove();
- }
- }
-
+ closeSession();
+ } catch (HiveClientException e) {
+ LOG.error("Unable to close Hive session: " + e.getMessage());
} finally {
transport.close();
transport = null;
client = null;
+ sessHandle = null;
protocol = null;
}
LOG.info("Connection to Hive closed");
@@ -288,29 +250,19 @@ public class Connection {
* @return handle of operation
* @throws HiveClientException
*/
- public TOperationHandle execute(final TSessionHandle session, final String cmd, final boolean async) throws HiveClientException {
+ public TOperationHandle execute(final String cmd, final boolean async) throws HiveClientException {
TOperationHandle handle = null;
-
- String[] commands = cmd.split(";");
- for(int i=0; i<commands.length; i++) {
- final String oneCmd = commands[i];
- final boolean lastCommand = i == commands.length-1;
+ for(final String oneCmd : cmd.split(";")) {
TExecuteStatementResp execResp = new HiveCall<TExecuteStatementResp>(this) {
@Override
public TExecuteStatementResp body() throws HiveClientException {
TExecuteStatementReq execReq = null;
- execReq = new TExecuteStatementReq(session, oneCmd);
-
- // only last command should be asynchronous and return some results
- // all previous commands are supposed to be set properties entries
- if (lastCommand) {
- execReq.setRunAsync(async);
- } else {
- execReq.setRunAsync(false);
- }
- execReq.setConfOverlay(new HashMap<String, String>());
+ openSession();
+ execReq = new TExecuteStatementReq(getSessHandle(), oneCmd);
+ execReq.setRunAsync(async);
+ execReq.setConfOverlay(new HashMap<String, String>()); //maybe it's hive configuration? use it, Luke!
try {
return client.ExecuteStatement(execReq);
} catch (TException e) {
@@ -330,12 +282,12 @@ public class Connection {
return handle;
}
- public TOperationHandle executeAsync(TSessionHandle session, String cmd) throws HiveClientException {
- return execute(session, cmd, true);
+ public TOperationHandle executeAsync(String cmd) throws HiveClientException {
+ return execute(cmd, true);
}
- public TOperationHandle executeSync(TSessionHandle session, String cmd) throws HiveClientException {
- return execute(session, cmd, false);
+ public TOperationHandle executeSync(String cmd) throws HiveClientException {
+ return execute(cmd, false);
}
public String getLogs(TOperationHandle handle) {
@@ -423,6 +375,16 @@ public class Connection {
this.host = host;
}
+ public TSessionHandle getSessHandle() throws HiveClientException {
+ if (sessHandle == null)
+ openSession();
+ return sessHandle;
+ }
+
+ public void setSessHandle(TSessionHandle sessHandle) {
+ this.sessHandle = sessHandle;
+ }
+
public TCLIService.Client getClient() {
return client;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/DDLDelegator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/DDLDelegator.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/DDLDelegator.java
index 8326e50..9cb5a1b 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/DDLDelegator.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/DDLDelegator.java
@@ -37,8 +37,8 @@ public class DDLDelegator {
* @return list of table names
* @throws HiveClientException
*/
- public List<String> getTableList(TSessionHandle session, String db, String like) throws HiveClientException {
- Cursor cursor = getTableListCursor(session, db, like);
+ public List<String> getTableList(String db, String like) throws HiveClientException {
+ Cursor cursor = getTableListCursor(db, like);
return cursor.getValuesInColumn(0);
}
@@ -48,9 +48,9 @@ public class DDLDelegator {
* @return list of table names
* @throws HiveClientException
*/
- public Cursor getTableListCursor(TSessionHandle session, String db, String like) throws HiveClientException {
- connection.executeSync(session, String.format("use %s", db));
- TOperationHandle handle = connection.executeSync(session, String.format("show tables like '%s'", like));
+ public Cursor getTableListCursor(String db, String like) throws HiveClientException {
+ connection.executeSync(String.format("use %s", db));
+ TOperationHandle handle = connection.executeSync(String.format("show tables like '%s'", like));
return new Cursor(connection, handle);
}
@@ -61,8 +61,8 @@ public class DDLDelegator {
* @return list of databases
* @throws HiveClientException
*/
- public List<String> getDBList(TSessionHandle session, String like) throws HiveClientException {
- Cursor cursor = getDBListCursor(session, like);
+ public List<String> getDBList(String like) throws HiveClientException {
+ Cursor cursor = getDBListCursor(like);
return cursor.getValuesInColumn(0);
}
@@ -72,8 +72,8 @@ public class DDLDelegator {
* @return list of databases
* @throws HiveClientException
*/
- public Cursor getDBListCursor(TSessionHandle session, String like) throws HiveClientException {
- TOperationHandle handle = connection.executeSync(session, String.format("show databases like '%s'", like));
+ public Cursor getDBListCursor(String like) throws HiveClientException {
+ TOperationHandle handle = connection.executeSync(String.format("show databases like '%s'", like));
return new Cursor(connection, handle);
}
@@ -84,9 +84,9 @@ public class DDLDelegator {
* @return schema
* @throws HiveClientException
*/
- public List<ColumnDescription> getTableDescription(TSessionHandle session, final String db, final String table, String like, boolean extended) throws HiveClientException {
+ public List<ColumnDescription> getTableDescription(final String db, final String table, String like, boolean extended) throws HiveClientException {
List<ColumnDescription> columnDescriptions = new LinkedList<ColumnDescription>();
- Cursor cursor = getTableDescriptionCursor(session, db, table, like);
+ Cursor cursor = getTableDescriptionCursor(db, table, like);
for(Row row : cursor) {
Object[] rowObjects = row.getRow();
@@ -112,7 +112,7 @@ public class DDLDelegator {
* @return schema
* @throws HiveClientException
*/
- public Cursor getTableDescriptionCursor(final TSessionHandle session, final String db, final String table, String like) throws HiveClientException {
+ public Cursor getTableDescriptionCursor(final String db, final String table, String like) throws HiveClientException {
if (like == null)
like = ".*";
else
@@ -122,7 +122,7 @@ public class DDLDelegator {
@Override
public TGetColumnsResp body() throws HiveClientException {
- TGetColumnsReq req = new TGetColumnsReq(session);
+ TGetColumnsReq req = new TGetColumnsReq(conn.getSessHandle());
req.setSchemaName(db);
req.setTableName(table);
req.setColumnName(finalLike);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
index 891b526..28cf4d1 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
@@ -36,15 +36,15 @@ abstract public class CRUDResourceManager<T extends Indexed> implements IResourc
private Storage storage = null;
protected final Class<? extends T> resourceClass;
- protected IStorageFactory storageFactory;
+ protected IStorageFactory storageFabric;
/**
* Constructor
* @param resourceClass model class
*/
- public CRUDResourceManager(Class<? extends T> resourceClass, IStorageFactory storageFactory) {
+ public CRUDResourceManager(Class<? extends T> resourceClass, IStorageFactory storageFabric) {
this.resourceClass = resourceClass;
- this.storageFactory = storageFactory;
+ this.storageFabric = storageFabric;
}
// CRUD operations
@@ -68,7 +68,7 @@ abstract public class CRUDResourceManager<T extends Indexed> implements IResourc
@Override
public T read(Object id) throws ItemNotFound {
T object = null;
- object = storageFactory.getStorage().load(this.resourceClass, id);
+ object = storageFabric.getStorage().load(this.resourceClass, id);
if (!checkPermissions(object))
throw new ItemNotFound();
return object;
@@ -81,7 +81,7 @@ abstract public class CRUDResourceManager<T extends Indexed> implements IResourc
*/
@Override
public List<T> readAll(FilteringStrategy filteringStrategy) {
- return storageFactory.getStorage().loadAll(this.resourceClass, filteringStrategy);
+ return storageFabric.getStorage().loadAll(this.resourceClass, filteringStrategy);
}
/**
@@ -105,16 +105,16 @@ abstract public class CRUDResourceManager<T extends Indexed> implements IResourc
*/
@Override
public void delete(Object resourceId) throws ItemNotFound {
- if (!storageFactory.getStorage().exists(this.resourceClass, resourceId)) {
+ if (!storageFabric.getStorage().exists(this.resourceClass, resourceId)) {
throw new ItemNotFound();
}
- storageFactory.getStorage().delete(this.resourceClass, resourceId);
+ storageFabric.getStorage().delete(this.resourceClass, resourceId);
}
// UTILS
protected T save(T object) {
- storageFactory.getStorage().store(resourceClass, object);
+ storageFabric.getStorage().store(resourceClass, object);
return object;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
index 4593b76..15fcf22 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
@@ -49,7 +49,7 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU
@Override
public T update(T newObject, String id) throws ItemNotFound {
- T object = storageFactory.getStorage().load(this.resourceClass, id);
+ T object = storageFabric.getStorage().load(this.resourceClass, id);
if (object.getOwner().compareTo(this.context.getUsername()) != 0) {
throw new ItemNotFound();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
index 844d20e..bb1a0a2 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
@@ -29,7 +29,6 @@ import org.apache.ambari.view.hive.utils.BadRequestFormattedException;
import org.apache.ambari.view.hive.utils.ServiceFormattedException;
import org.apache.ambari.view.hive.utils.SharedObjectsFactory;
import org.apache.commons.collections4.map.PassiveExpiringMap;
-import org.apache.hive.service.cli.thrift.TSessionHandle;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,8 +89,7 @@ public class HiveBrowserService {
String curl = null;
try {
JSONObject response = new JSONObject();
- TSessionHandle session = getConnectionFactory().getHiveConnection().getOrCreateSessionByTag("DDL");
- List<String> tables = getConnectionFactory().getHiveConnection().ddl().getDBList(session, like);
+ List<String> tables = getConnectionFactory().getHiveConnection().ddl().getDBList(like);
response.put("databases", tables);
return Response.ok(response).build();
} catch (WebApplicationException ex) {
@@ -126,8 +124,7 @@ public class HiveBrowserService {
new Callable<Cursor>() {
@Override
public Cursor call() throws Exception {
- TSessionHandle session = getConnectionFactory().getHiveConnection().getOrCreateSessionByTag("DDL");
- return getConnectionFactory().getHiveConnection().ddl().getDBListCursor(session, finalLike);
+ return getConnectionFactory().getHiveConnection().ddl().getDBListCursor(finalLike);
}
}).build();
} catch (WebApplicationException ex) {
@@ -157,8 +154,7 @@ public class HiveBrowserService {
String curl = null;
try {
JSONObject response = new JSONObject();
- TSessionHandle session = getConnectionFactory().getHiveConnection().getOrCreateSessionByTag("DDL");
- List<String> tables = getConnectionFactory().getHiveConnection().ddl().getTableList(session, db, like);
+ List<String> tables = getConnectionFactory().getHiveConnection().ddl().getTableList(db, like);
response.put("tables", tables);
response.put("database", db);
return Response.ok(response).build();
@@ -195,8 +191,7 @@ public class HiveBrowserService {
new Callable<Cursor>() {
@Override
public Cursor call() throws Exception {
- TSessionHandle session = getConnectionFactory().getHiveConnection().getOrCreateSessionByTag("DDL");
- Cursor cursor = getConnectionFactory().getHiveConnection().ddl().getTableListCursor(session, db, finalLike);
+ Cursor cursor = getConnectionFactory().getHiveConnection().ddl().getTableListCursor(db, finalLike);
cursor.selectColumns(requestedColumns);
return cursor;
}
@@ -225,9 +220,8 @@ public class HiveBrowserService {
String curl = null;
try {
JSONObject response = new JSONObject();
- TSessionHandle session = getConnectionFactory().getHiveConnection().getOrCreateSessionByTag("DDL");
List<ColumnDescription> columnDescriptions = getConnectionFactory().getHiveConnection().ddl()
- .getTableDescription(session, db, table, like, extendedTableDescription);
+ .getTableDescription(db, table, like, extendedTableDescription);
response.put("columns", columnDescriptions);
response.put("database", db);
response.put("table", table);
@@ -261,9 +255,7 @@ public class HiveBrowserService {
new Callable<Cursor>() {
@Override
public Cursor call() throws Exception {
- TSessionHandle session = getConnectionFactory().getHiveConnection().getOrCreateSessionByTag("DDL");
- Cursor cursor = getConnectionFactory().getHiveConnection().ddl().
- getTableDescriptionCursor(session, db, table, like);
+ Cursor cursor = getConnectionFactory().getHiveConnection().ddl().getTableDescriptionCursor(db, table, like);
cursor.selectColumns(requestedColumns);
return cursor;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
index e5fae95..ce82e15 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
@@ -30,8 +30,6 @@ import org.apache.ambari.view.hive.resources.jobs.viewJobs.Job;
import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl;
import org.apache.commons.beanutils.PropertyUtils;
import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.hive.service.cli.thrift.TOperationHandle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,22 +40,8 @@ import java.util.List;
import java.util.Set;
/**
- * View Jobs and ATS Jobs aggregator.
- * There are 4 options:
- * 1) ATS Job without operationId
- * *Meaning*: executed outside of HS2
- * - Job info only from ATS
- * 2) ATS Job with operationId
- * a) Hive View Job with same operationId is not present
- * *Meaning*: executed with HS2
- * - Job info only from ATS
- * b) Hive View Job with operationId is present (need to merge)
- * *Meaning*: executed with HS2 through Hive View
- * - Job info merged from ATS and from Hive View DataStorage
- * 3) Job present only in Hive View, ATS does not have it
- * *Meaning*: executed through Hive View, but Hadoop Job was not created
- * it can happen if user executes query without aggregation, like just "select * from TABLE"
- * - Job info only from Hive View
+ * View Jobs and ATS Jobs aggregator
+ * Not all ViewJobs create ATS job
*/
public class Aggregator {
protected final static Logger LOG =
@@ -79,7 +63,7 @@ public class Aggregator {
Set<String> addedOperationIds = new HashSet<String>();
List<Job> allJobs = new LinkedList<Job>();
- for (HiveQueryId atsHiveQuery : ats.getHiveQueryIdsList(username)) {
+ for (HiveQueryId atsHiveQuery : ats.getHiveQuieryIdsList(username)) {
TezDagId atsTezDag;
if (atsHiveQuery.dagNames != null && atsHiveQuery.dagNames.size() > 0) {
@@ -128,25 +112,6 @@ public class Aggregator {
return allJobs;
}
- public Job readATSJob(Job viewJob) throws ItemNotFound {
- TOperationHandle operationHandle = operationHandleResourceManager.getHandleForJob(viewJob);
-
- String hexGuid = Hex.encodeHexString(operationHandle.getOperationId().getGuid());
- HiveQueryId atsHiveQuery = ats.getHiveQueryIdByOperationId(hexStringToUrlSafeBase64(hexGuid));
-
- TezDagId atsTezDag;
- if (atsHiveQuery.dagNames != null && atsHiveQuery.dagNames.size() > 0) {
- String dagName = atsHiveQuery.dagNames.get(0);
-
- atsTezDag = ats.getTezDAGByName(dagName);
- } else {
- atsTezDag = new TezDagId();
- }
-
- saveJobInfoIfNeeded(atsHiveQuery, atsTezDag, viewJob);
- return mergeAtsJobWithViewJob(atsHiveQuery, atsTezDag, viewJob);
- }
-
protected boolean hasOperationId(HiveQueryId atsHiveQuery) {
return atsHiveQuery.operationId != null;
}
@@ -170,15 +135,11 @@ public class Aggregator {
}
protected void saveJobInfoIfNeeded(HiveQueryId hiveQueryId, TezDagId tezDagId, Job viewJob) throws ItemNotFound {
- if (viewJob.getDagName() == null || viewJob.getDagName().isEmpty()) {
- if (hiveQueryId.dagNames != null && hiveQueryId.dagNames.size() > 0) {
- viewJob.setDagName(hiveQueryId.dagNames.get(0));
- viewJob.setDagId(tezDagId.entity);
- viewJobResourceManager.update(viewJob, viewJob.getId());
- }
+ if (viewJob.getDagName() == null) {
+ viewJob.setDagName(tezDagId.dagName);
+ viewJobResourceManager.update(viewJob, viewJob.getId());
}
- if ((tezDagId.status.compareToIgnoreCase(Job.JOB_STATE_UNKNOWN) != 0) &&
- !viewJob.getStatus().equals(tezDagId.status)) {
+ if (viewJob.getStatus().equals(tezDagId.status)) {
viewJob.setStatus(tezDagId.status);
viewJobResourceManager.update(viewJob, viewJob.getId());
}
@@ -199,10 +160,8 @@ public class Aggregator {
protected JobImpl fillAtsJobFields(JobImpl atsJob, HiveQueryId atsHiveQuery, TezDagId atsTezDag) {
atsJob.setApplicationId(atsTezDag.applicationId);
- if (atsHiveQuery.dagNames != null && atsHiveQuery.dagNames.size() > 0)
- atsJob.setDagName(atsHiveQuery.dagNames.get(0));
- atsJob.setDagId(atsTezDag.entity);
- if (atsTezDag.status != null && !atsTezDag.status.equals(TezDagId.STATUS_UNKNOWN))
+ atsJob.setDagName(atsTezDag.dagName);
+ if (!atsTezDag.status.equals(TezDagId.STATUS_UNKNOWN))
atsJob.setStatus(atsTezDag.status);
if (atsHiveQuery.starttime != 0)
atsJob.setDateSubmitted(atsHiveQuery.starttime);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
index c52d7b5..b70abe2 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
@@ -21,9 +21,7 @@ package org.apache.ambari.view.hive.resources.jobs;
import org.apache.ambari.view.hive.client.Connection;
import org.apache.ambari.view.hive.client.HiveClientException;
import org.apache.ambari.view.hive.utils.ServiceFormattedException;
-import org.apache.commons.codec.binary.Hex;
import org.apache.hive.service.cli.thrift.TOperationHandle;
-import org.apache.hive.service.cli.thrift.TSessionHandle;
public class ConnectionController {
@@ -35,39 +33,14 @@ public class ConnectionController {
this.operationHandleControllerFactory = operationHandleControllerFactory;
}
- public TSessionHandle getSessionByTag(String tag) {
- try {
- return connection.getSessionByTag(tag);
- } catch (HiveClientException e) {
- throw new ServiceFormattedException(e.toString(), e);
- }
- }
-
- public String openSession() {
- try {
- TSessionHandle sessionHandle = connection.openSession();
- return getTagBySession(sessionHandle);
- } catch (HiveClientException e) {
- throw new ServiceFormattedException(e.toString(), e);
- }
- }
-
- public static String getTagBySession(TSessionHandle sessionHandle) {
- return Hex.encodeHexString(sessionHandle.getSessionId().getGuid());
- }
-
- public void selectDatabase(TSessionHandle session, String database) {
- try {
- connection.executeSync(session, "use " + database + ";");
- } catch (HiveClientException e) {
- throw new ServiceFormattedException(e.toString(), e);
- }
+ public void selectDatabase(String database) {
+ executeQuery("use " + database + ";");
}
- public OperationHandleController executeQuery(TSessionHandle session, String cmd) {
+ public OperationHandleController executeQuery(String cmd) {
TOperationHandle operationHandle = null;
try {
- operationHandle = connection.executeAsync(session, cmd);
+ operationHandle = connection.executeAsync(cmd);
} catch (HiveClientException e) {
throw new ServiceFormattedException(e.toString(), e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobService.java
index 1a6f7bf..d9c69e8 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobService.java
@@ -66,7 +66,6 @@ public class JobService extends BaseService {
private IOperationHandleResourceManager opHandleResourceManager;
protected final static Logger LOG =
LoggerFactory.getLogger(JobService.class);
- private Aggregator aggregator;
protected synchronized JobResourceManager getResourceManager() {
if (resourceManager == null) {
@@ -76,26 +75,13 @@ public class JobService extends BaseService {
return resourceManager;
}
- protected IOperationHandleResourceManager getOperationHandleResourceManager() {
+ private IOperationHandleResourceManager getOperationHandleResourceManager() {
if (opHandleResourceManager == null) {
opHandleResourceManager = new OperationHandleResourceManager(getSharedObjectsFactory());
}
return opHandleResourceManager;
}
- protected Aggregator getAggregator() {
- if (aggregator == null) {
- ATSRequestsDelegate transport = new ATSRequestsDelegateImpl(context, "http://127.0.0.1:8188");
- IATSParser atsParser = new ATSParser(transport);
- aggregator = new Aggregator(getResourceManager(), getOperationHandleResourceManager(), atsParser);
- }
- return aggregator;
- }
-
- protected void setAggregator(Aggregator aggregator) {
- this.aggregator = aggregator;
- }
-
/**
* Get single item
*/
@@ -119,15 +105,7 @@ public class JobService extends BaseService {
}
private JSONObject jsonObjectFromJob(JobController jobController) throws IllegalAccessException, NoSuchMethodException, InvocationTargetException {
- Job hiveJob = jobController.getJobPOJO();
-
- Job mergedJob;
- try {
- mergedJob = getAggregator().readATSJob(hiveJob);
- } catch (ItemNotFound itemNotFound) {
- throw new ServiceFormattedException("Job not found", itemNotFound);
- }
- Map createdJobMap = PropertyUtils.describe(mergedJob);
+ Map createdJobMap = PropertyUtils.describe(jobController.getJob());
createdJobMap.remove("class"); // no need to show Bean class on client
JSONObject jobJson = new JSONObject();
@@ -338,10 +316,10 @@ public class JobService extends BaseService {
public Response getList() {
try {
LOG.debug("Getting all job");
- List<Job> allJobs = getAggregator().readAll(context.getUsername());
- for(Job job : allJobs) {
- job.setSessionTag(null);
- }
+ ATSRequestsDelegate transport = new ATSRequestsDelegateImpl(context, "http://127.0.0.1:8188");
+ IATSParser atsParser = new ATSParser(transport);
+ Aggregator aggregator = new Aggregator(getResourceManager(), getOperationHandleResourceManager(), atsParser);
+ List allJobs = aggregator.readAll(context.getUsername());
JSONObject object = new JSONObject();
object.put("jobs", allJobs);
@@ -367,7 +345,6 @@ public class JobService extends BaseService {
JobController createdJobController = getResourceManager().readController(job.getId());
createdJobController.submit();
- getResourceManager().saveIfModified(createdJobController);
response.setHeader("Location",
String.format("%s/%s", ui.getAbsolutePath().toString(), job.getId()));
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/OperationHandleResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/OperationHandleResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/OperationHandleResourceManager.java
index bab30a0..5004677 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/OperationHandleResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/OperationHandleResourceManager.java
@@ -40,7 +40,7 @@ public class OperationHandleResourceManager extends SharedCRUDResourceManager<St
@Override
public List<StoredOperationHandle> readJobRelatedHandles(final Job job) {
- return storageFactory.getStorage().loadAll(StoredOperationHandle.class, new FilteringStrategy() {
+ return storageFabric.getStorage().loadAll(StoredOperationHandle.class, new FilteringStrategy() {
@Override
public boolean isConform(Indexed item) {
StoredOperationHandle handle = (StoredOperationHandle) item;
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/ATSParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/ATSParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/ATSParser.java
index 106babd..b644d4c 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/ATSParser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/ATSParser.java
@@ -18,6 +18,7 @@
package org.apache.ambari.view.hive.resources.jobs.atsJobs;
+import org.apache.ambari.view.hive.utils.ServiceFormattedException;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
@@ -40,7 +41,7 @@ public class ATSParser implements IATSParser {
}
@Override
- public List<HiveQueryId> getHiveQueryIdsList(String username) {
+ public List<HiveQueryId> getHiveQuieryIdsList(String username) {
JSONObject entities = delegate.hiveQueryIdList(username);
JSONArray jobs = (JSONArray) entities.get("entities");
@@ -58,13 +59,15 @@ public class ATSParser implements IATSParser {
}
@Override
- public HiveQueryId getHiveQueryIdByOperationId(String guidString) {
+ public HiveQueryId getHiveQuieryIdByOperationId(byte[] guid) {
+ String guidString = new String(guid);
JSONObject entities = delegate.hiveQueryIdByOperationId(guidString);
JSONArray jobs = (JSONArray) entities.get("entities");
assert jobs.size() <= 1;
if (jobs.size() == 0) {
- return new HiveQueryId();
+ //TODO: throw appropriate exception
+ throw new ServiceFormattedException("HIVE_QUERY_ID with operationid=" + guidString + " not found");
}
return parseAtsHiveJob((JSONObject) jobs.get(0));
@@ -81,7 +84,6 @@ public class ATSParser implements IATSParser {
TezDagId parsedDag = new TezDagId();
JSONArray applicationIds = (JSONArray) ((JSONObject) tezDagEntity.get("primaryfilters")).get("applicationId");
- parsedDag.entity = (String) tezDagEntity.get("entity");
parsedDag.applicationId = (String) applicationIds.get(0);
parsedDag.status = (String) ((JSONObject) tezDagEntity.get("otherinfo")).get("status");
return parsedDag;
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/IATSParser.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/IATSParser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/IATSParser.java
index 6109001..d029fdc 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/IATSParser.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/IATSParser.java
@@ -21,9 +21,9 @@ package org.apache.ambari.view.hive.resources.jobs.atsJobs;
import java.util.List;
public interface IATSParser {
- List<HiveQueryId> getHiveQueryIdsList(String username);
+ List<HiveQueryId> getHiveQuieryIdsList(String username);
- HiveQueryId getHiveQueryIdByOperationId(String guidString);
+ HiveQueryId getHiveQuieryIdByOperationId(byte[] guid);
TezDagId getTezDAGByName(String name);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/TezDagId.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/TezDagId.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/TezDagId.java
index a814286..061c51c 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/TezDagId.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/atsJobs/TezDagId.java
@@ -21,6 +21,6 @@ package org.apache.ambari.view.hive.resources.jobs.atsJobs;
public class TezDagId {
public static final String STATUS_UNKNOWN = "UNKNOWN";
public String applicationId = "";
- public String entity = "";
+ public String dagName = "";
public String status = STATUS_UNKNOWN;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/Job.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/Job.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/Job.java
index 65c36a7..004932c 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/Job.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/Job.java
@@ -95,13 +95,5 @@ public interface Job extends Serializable,Indexed,PersonalResource {
String getDagName();
- void setDagName(String dagName);
-
- String getDagId();
-
- void setDagId(String dagId);
-
- String getSessionTag();
-
- void setSessionTag(String sessionTag);
+ void setDagName(String DagName);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobControllerImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobControllerImpl.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobControllerImpl.java
index c6c1c78..a100f3d 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobControllerImpl.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobControllerImpl.java
@@ -28,7 +28,6 @@ import org.apache.ambari.view.hive.resources.savedQueries.SavedQueryResourceMana
import org.apache.ambari.view.hive.utils.*;
import org.apache.ambari.view.hive.utils.HdfsApi;
import org.apache.ambari.view.hive.utils.HdfsUtil;
-import org.apache.hive.service.cli.thrift.TSessionHandle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +47,7 @@ public class JobControllerImpl implements JobController, ModifyNotificationDeleg
private boolean modified;
private OperationHandleControllerFactory opHandleControllerFactory;
- private ConnectionController hiveConnection;
+ private ConnectionController hiveSession;
private SavedQueryResourceManager savedQueryResourceManager;
private IATSParser atsParser;
@@ -57,7 +56,7 @@ public class JobControllerImpl implements JobController, ModifyNotificationDeleg
* Warning: Create JobControllers ONLY using JobControllerFactory!
*/
public JobControllerImpl(ViewContext context, Job job,
- ConnectionController hiveConnection,
+ ConnectionController hiveSession,
OperationHandleControllerFactory opHandleControllerFactory,
SavedQueryResourceManager savedQueryResourceManager,
IATSParser atsParser,
@@ -65,7 +64,7 @@ public class JobControllerImpl implements JobController, ModifyNotificationDeleg
this.context = context;
setJobPOJO(job);
this.opHandleControllerFactory = opHandleControllerFactory;
- this.hiveConnection = hiveConnection;
+ this.hiveSession = hiveSession;
this.savedQueryResourceManager = savedQueryResourceManager;
this.atsParser = atsParser;
this.hdfsApi = hdfsApi;
@@ -98,24 +97,16 @@ public class JobControllerImpl implements JobController, ModifyNotificationDeleg
setupHiveBeforeQueryExecute();
String query = getQueryForJob();
- OperationHandleController handleController = hiveConnection.executeQuery(getSession(), query);
+ OperationHandleController handleController = hiveSession.executeQuery(query);
handleController.persistHandleForJob(job);
+
+// atsParser.getHiveQuieryIdsList()
}
private void setupHiveBeforeQueryExecute() {
String database = getJobDatabase();
- hiveConnection.selectDatabase(getSession(), database);
- }
-
- private TSessionHandle getSession() {
- if (job.getSessionTag() != null) {
- return hiveConnection.getSessionByTag(getJob().getSessionTag());
- } else {
- String tag = hiveConnection.openSession();
- job.setSessionTag(tag);
- return hiveConnection.getSessionByTag(tag);
- }
+ hiveSession.selectDatabase(database);
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobImpl.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobImpl.java
index d0d74a2..1858b3e 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobImpl.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobImpl.java
@@ -38,11 +38,8 @@ public class JobImpl implements Job {
private String queryId = null;
private String applicationId;
- private String dagId;
private String dagName;
- private String sessionTag;
-
private String id = null;
private String owner = null;
@@ -223,27 +220,7 @@ public class JobImpl implements Job {
}
@Override
- public void setDagName(String dagName) {
- this.dagName = dagName;
- }
-
- @Override
- public String getDagId() {
- return dagId;
- }
-
- @Override
- public void setDagId(String dagId) {
- this.dagId = dagId;
- }
-
- @Override
- public String getSessionTag() {
- return sessionTag;
- }
-
- @Override
- public void setSessionTag(String sessionTag) {
- this.sessionTag = sessionTag;
+ public void setDagName(String DagName) {
+ this.dagName = DagName;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobResourceManager.java
index ed1866e..101e328 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/viewJobs/JobResourceManager.java
@@ -30,7 +30,7 @@ import org.slf4j.LoggerFactory;
import java.util.*;
/**
- * Object that provides CRUD operations for job objects
+ * Object that provides CRUD operations for query objects
*/
public class JobResourceManager extends PersonalCRUDResourceManager<Job> {
private final static Logger LOG =
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/savedQueries/SavedQueryResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/savedQueries/SavedQueryResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/savedQueries/SavedQueryResourceManager.java
index 01f9c9c..c032bb1 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/savedQueries/SavedQueryResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/savedQueries/SavedQueryResourceManager.java
@@ -19,6 +19,7 @@
package org.apache.ambari.view.hive.resources.savedQueries;
import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
import org.apache.ambari.view.hive.persistence.utils.FilteringStrategy;
import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
import org.apache.ambari.view.hive.resources.PersonalCRUDResourceManager;
@@ -30,7 +31,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Object that provides CRUD operations for query objects
@@ -82,7 +85,7 @@ public class SavedQueryResourceManager extends PersonalCRUDResourceManager<Saved
HdfsUtil.putStringToFile(sharedObjectsFactory.getHdfsApi(), newFilePath, "");
object.setQueryFile(newFilePath);
- storageFactory.getStorage().store(SavedQuery.class, object);
+ storageFabric.getStorage().store(SavedQuery.class, object);
}
@Override
@@ -107,7 +110,7 @@ public class SavedQueryResourceManager extends PersonalCRUDResourceManager<Saved
}
savedQuery.setShortQuery(query.substring(0, (query.length() > 42)?42:query.length()));
}
- storageFactory.getStorage().store(SavedQuery.class, savedQuery);
+ storageFabric.getStorage().store(SavedQuery.class, savedQuery);
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/index.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/index.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/index.js
index 954b42b..592bb06 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/index.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/index.js
@@ -42,10 +42,6 @@ export default Ember.Controller.extend({
visualExplain: Ember.computed.alias('controllers.' + constants.namingConventions.visualExplain),
tezUI: Ember.computed.alias('controllers.' + constants.namingConventions.tezUI),
- shouldShowTez: function() {
- return this.get('model.dagId') && this.get('tezUI.isTezViewAvailable');
- }.property('model.dagId', 'tezUI.isTezViewAvailable'),
-
canExecute: function () {
var isModelRunning = this.get('model.isRunning');
var hasParams = this.get('queryParams.length');
@@ -93,7 +89,6 @@ export default Ember.Controller.extend({
job = this.store.createRecord(constants.namingConventions.job, {
title: originalModel.get('title'),
- sessionTag: originalModel.get('sessionTag'),
dataBase: this.get('databases.selectedDatabase.name')
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/settings.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/settings.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/settings.js
index 6ac0828..c59fc88 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/settings.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/settings.js
@@ -84,30 +84,23 @@ export default Ember.ArrayController.extend({
return;
}
- var Setting = Ember.Object.extend({
- key: Ember.Object.create(),
- valid: true,
- selection: Ember.Object.create(),
- value: Ember.computed.alias('selection.value')
- });
-
query.set('fileContent', content.replace(regex, '').trim());
settings = settings.map(function (setting) {
var KV = setting.split('=');
- var name = KV[0].replace('set', '').trim();
- var value = KV[1].replace(';', '').trim();
-
- var newSetting = Setting.create({});
- newSetting.set('key.name', name);
- newSetting.set('selection.value', value);
-
- if (!self.get('predefinedSettings').findBy('name', name)) {
+ var obj = {
+ key: {
+ name: KV[0].replace('set', '').trim()
+ },
+ value: KV[1].replace(';', '').trim()
+ };
+
+ if (!self.get('predefinedSettings').findBy('name', obj.key.name)) {
self.get('predefinedSettings').pushObject({
- name: name
+ name: obj.key.name
});
}
- return newSetting;
+ return obj;
});
this.setSettingForQuery(id, settings);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/tez-ui.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/tez-ui.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/tez-ui.js
index c511e0b..dc99fd1 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/tez-ui.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/controllers/tez-ui.js
@@ -17,90 +17,6 @@
*/
import Ember from 'ember';
-import constants from 'hive/utils/constants';
export default Ember.Controller.extend({
- needs: [ constants.namingConventions.index ],
-
- index: Ember.computed.alias('controllers.' + constants.namingConventions.index),
-
- tezViewURL: null,
- tezApiURL: '/api/v1/views/TEZ',
- tezURLPrefix: '/views/TEZ',
- tezDagPath: '?viewPath=/#/dag/',
-
- isTezViewAvailable: Ember.computed.bool('tezViewURL'),
-
- dagId: function() {
- if (this.get('isTezViewAvailable')) {
- return this.get('index.model.dagId');
- }
-
- return false;
- }.property('index.model.dagId', 'isTezViewAvailable'),
-
- dagURL: function() {
- if (this.get('dagId')) {
- return "%@%@%@".fmt(this.get('tezViewURL'), this.get('tezDagPath'), this.get('dagId'));
- }
-
- return false;
- }.property('dagId'),
-
- getTezView: function() {
- if (this.get('isTezViewAvailable')) {
- return;
- }
-
- var self = this;
- Ember.$.getJSON(this.get('tezApiURL'))
- .then(function(response) {
- self.getTezViewInstance(response);
- })
- .fail(function(response) {
- self.setTezViewError(response);
- });
- }.on('init'),
-
- getTezViewInstance: function(data) {
- var self = this;
- var url = data.versions[0].href;
-
- Ember.$.getJSON(url)
- .then(function(response) {
- if (!response.instances.length) {
- self.setTezViewError(response);
- return;
- }
-
- self.set('isTezViewAvailable', true);
-
- var instance = response.instances[0].ViewInstanceInfo;
- self.setTezViewURL(instance);
- });
- },
-
- setTezViewURL: function(instance) {
- var url = "%@/%@/%@".fmt(
- this.get('tezURLPrefix'),
- instance.version,
- instance.instance_name
- );
-
- this.set('tezViewURL', url);
- },
-
- setTezViewError: function(data) {
- // status: 404 => Tev View isn't deployed
- if (data.status && data.status === 404) {
- this.set('error', 'tez.errors.not.deployed');
- return;
- }
-
- // no instance created
- if (data.instances && !data.instances.length) {
- this.set('error', 'tez.errors.no.instance');
- return;
- }
- }
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
index 8d60248..a87bf78 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/initializers/i18n.js
@@ -159,11 +159,5 @@ TRANSLATIONS = {
}
},
add: 'Add'
- },
- tez: {
- errors: {
- 'not.deployed': "Tez View isn't deployed.",
- 'no.instance': "No instance of Tez View found."
- }
}
};
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/models/job.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/models/job.js b/contrib/views/hive/src/main/resources/ui/hive-web/app/models/job.js
index 06bdc2e..73fcc20 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/models/job.js
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/models/job.js
@@ -29,9 +29,6 @@ export default DS.Model.extend({
dateSubmitted: DS.attr('date'),
forcedContent: DS.attr('string'),
logFile: DS.attr('string'),
- dagName: DS.attr('string'),
- dagId: DS.attr('string'),
- sessionTag: DS.attr('string'),
dateSubmittedTimestamp: function () {
var date = this.get('dateSubmitted');
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss b/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
index 3f49713..7085c85 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/styles/app.scss
@@ -371,9 +371,3 @@ tabs {
tree-view ul li {
padding-left: 10px;
}
-
-#tez-ui iframe {
- width: 100%;
- height: 822px;
- border: none;
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/index.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/index.hbs b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/index.hbs
index 8621baa..b8d1d3b 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/index.hbs
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/index.hbs
@@ -97,6 +97,6 @@
<span {{bind-attr class="visualExplain.showOverlay:active :fa :fa-bar-chart :queries-icon"}} {{action 'toggleOverlay' 'visualExplain'}}></span>
- <span {{bind-attr class="tezUI.showOverlay:active shouldShowTez::hide :queries-icon :text-icon"}} {{action 'toggleOverlay' 'tezUI'}}>TEZ</span>
+ <span {{bind-attr class="tezUI.showOverlay:active :queries-icon :text-icon"}} {{action 'toggleOverlay' 'tezUI'}}>TEZ</span>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/tez-ui.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/tez-ui.hbs b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/tez-ui.hbs
index 4bee4d6..f9b8d9c 100644
--- a/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/tez-ui.hbs
+++ b/contrib/views/hive/src/main/resources/ui/hive-web/app/templates/tez-ui.hbs
@@ -18,12 +18,5 @@
<div id="tez-ui">
{{#panel-widget headingTranslation="titles.query.tez"}}
- {{#if dagURL}}
- <iframe {{bind-attr src=dagURL}}></iframe>
- {{else}}
- {{#if error}}
- <div class="alert alert-danger" role="alert"><strong>{{tb-helper error}}</strong></div>
- {{/if}}
- {{/if}}
{{/panel-widget}}
-</div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/ATSParserTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/ATSParserTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/ATSParserTest.java
index d2ff02c..ced7772 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/ATSParserTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/ATSParserTest.java
@@ -39,7 +39,7 @@ public class ATSParserTest {
public void testGetHiveJobsList() throws Exception {
IATSParser jobLoader = new ATSParser(new ATSRequestsDelegateStub());
- List<HiveQueryId> jobs = jobLoader.getHiveQueryIdsList("hive");
+ List<HiveQueryId> jobs = jobLoader.getHiveQuieryIdsList("hive");
Assert.assertEquals(1, jobs.size());
@@ -62,7 +62,6 @@ public class ATSParserTest {
TezDagId tezDag = jobLoader.getTezDAGByName("hive_20150209144848_c3a5a07b-c3b6-4f57-a6d5-3dadecdd6fd0:4");
- Assert.assertEquals("dag_1423156117563_0005_2", tezDag.entity);
Assert.assertEquals("application_1423156117563_0005", tezDag.applicationId);
Assert.assertEquals("SUCCEEDED", tezDag.status);
}
@@ -409,4 +408,4 @@ public class ATSParserTest {
);
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/AggregatorTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/AggregatorTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/AggregatorTest.java
index 50642ac..08e8b81 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/AggregatorTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/AggregatorTest.java
@@ -371,18 +371,18 @@ public class AggregatorTest {
}
@Override
- public List<HiveQueryId> getHiveQueryIdsList(String username) {
+ public List<HiveQueryId> getHiveQuieryIdsList(String username) {
return hiveQueryIds;
}
@Override
- public HiveQueryId getHiveQueryIdByOperationId(String guid) {
- return new HiveQueryId();
+ public HiveQueryId getHiveQuieryIdByOperationId(byte[] guid) {
+ return null;
}
@Override
public TezDagId getTezDAGByName(String name) {
- return new TezDagId();
+ return null;
}
public List<HiveQueryId> getHiveQueryIds() {
@@ -393,4 +393,4 @@ public class AggregatorTest {
this.hiveQueryIds = hiveQueryIds;
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c29dfa4/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/JobServiceTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/JobServiceTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/JobServiceTest.java
index 3c4202a..1c3444e 100644
--- a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/JobServiceTest.java
+++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/jobs/JobServiceTest.java
@@ -71,12 +71,6 @@ public class JobServiceTest extends BaseHiveTest {
Connection hiveConnection = configureHiveConnectionMock();
jobService.getSharedObjectsFactory().setInstance(Connection.class, hiveConnection);
- jobService.setAggregator(
- new Aggregator(
- jobService.getResourceManager(),
- jobService.getOperationHandleResourceManager(),
- new AggregatorTest.MockATSParser())
- );
}
@Test
@@ -136,17 +130,14 @@ public class JobServiceTest extends BaseHiveTest {
ServiceTestUtils.getResponseWithLocation(), ServiceTestUtils.getDefaultUriInfo());
}
+
+
private Connection configureHiveConnectionMock() throws HiveClientException {
TGetOperationStatusResp statusResp = getOperationStatusResp();
TOperationHandle operationHandle = getExecutionOperationHandle();
Connection connection = createNiceMock(Connection.class);
- TSessionHandle sessionHandle = new TSessionHandle();
- THandleIdentifier handleIdentifier = new THandleIdentifier();
- handleIdentifier.setGuid(new byte[]{1,2,3,4,5,6,7,8});
- sessionHandle.setSessionId(handleIdentifier);
- expect(connection.openSession()).andReturn(sessionHandle).anyTimes();
- expect(connection.executeAsync((TSessionHandle) anyObject(), anyString())).andReturn(operationHandle).anyTimes();
+ expect(connection.executeAsync(anyString())).andReturn(operationHandle).anyTimes();
expect(connection.getLogs(anyObject(TOperationHandle.class))).andReturn("some logs").anyTimes();
expect(connection.getOperationStatus(anyObject(TOperationHandle.class))).andReturn(statusResp).anyTimes();