You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ni...@apache.org on 2016/07/18 11:52:57 UTC
ambari git commit: AMBARI-17605. fixed Upload table using Hive view
will fail if Hive database is created with Location option (nitirajrathore)
Repository: ambari
Updated Branches:
refs/heads/trunk a12032d76 -> 7c0357c49
AMBARI-17605. fixed Upload table using Hive view will fail if Hive database is created with Location option (nitirajrathore)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7c0357c4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7c0357c4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7c0357c4
Branch: refs/heads/trunk
Commit: 7c0357c49273782933e6456d2f694bc1c114be58
Parents: a12032d
Author: Nitiraj Rathore <ni...@gmail.com>
Authored: Mon Jul 18 17:21:03 2016 +0530
Committer: Nitiraj Rathore <ni...@gmail.com>
Committed: Mon Jul 18 17:21:03 2016 +0530
----------------------------------------------------------------------
.../ambari/view/hive2/client/DDLDelegator.java | 2 +
.../view/hive2/client/DDLDelegatorImpl.java | 13 +++++
.../resources/browser/HiveBrowserService.java | 4 +-
.../hive2/resources/uploads/UploadService.java | 61 +++++++++++++++-----
4 files changed, 64 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c0357c4/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java
index 20b91f1..1774a65 100644
--- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java
+++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java
@@ -26,6 +26,8 @@ public interface DDLDelegator {
List<String> getTableList(ConnectionConfig config, String database, String like);
+ List<Row> getTableDescriptionFormatted(ConnectionConfig config, String database, String table);
+
List<ColumnDescription> getTableDescription(ConnectionConfig config, String database, String table, String like, boolean extended);
Cursor<Row, ColumnDescription> getDbListCursor(ConnectionConfig config, String like);
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c0357c4/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java
index 0171626..dff9471 100644
--- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java
+++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java
@@ -81,6 +81,19 @@ public class DDLDelegatorImpl implements DDLDelegator {
}
@Override
+ public List<Row> getTableDescriptionFormatted(ConnectionConfig config, String database, String table) {
+ Optional<Result> rowsFromDB = getRowsFromDB(config, getTableDescriptionStatements(database, table));
+ return rowsFromDB.isPresent() ? rowsFromDB.get().getRows() : null;
+ }
+
+ private String[] getTableDescriptionStatements(String database, String table) {
+ return new String[]{
+ String.format("use %s",database),
+ String.format("describe formatted %s", table)
+ };
+ }
+
+ @Override
public List<ColumnDescription> getTableDescription(ConnectionConfig config, String database, String table, String like, boolean extended) {
Optional<Result> resultOptional = getTableDescription(config, database, table, like);
List<ColumnDescription> descriptions = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c0357c4/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java
index 0da49c4..7481f9e 100644
--- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java
+++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java
@@ -20,6 +20,8 @@ package org.apache.ambari.view.hive2.resources.browser;
import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.ViewResourceHandler;
+import org.apache.ambari.view.hive2.ConnectionFactory;
+import org.apache.ambari.view.hive2.ConnectionSystem;
import org.apache.ambari.view.hive2.client.ColumnDescription;
import org.apache.ambari.view.hive2.client.ConnectionConfig;
import org.apache.ambari.view.hive2.client.Cursor;
@@ -29,8 +31,6 @@ import org.apache.ambari.view.hive2.client.Row;
import org.apache.ambari.view.hive2.resources.jobs.ResultsPaginationController;
import org.apache.ambari.view.hive2.utils.BadRequestFormattedException;
import org.apache.ambari.view.hive2.utils.ServiceFormattedException;
-import org.apache.ambari.view.hive2.ConnectionFactory;
-import org.apache.ambari.view.hive2.ConnectionSystem;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c0357c4/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java
index 18da1ff..08d91e9 100644
--- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java
+++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java
@@ -20,8 +20,14 @@ package org.apache.ambari.view.hive2.resources.uploads;
import com.sun.jersey.core.header.FormDataContentDisposition;
import com.sun.jersey.multipart.FormDataParam;
+import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.hive.resources.uploads.CSVParams;
import org.apache.ambari.view.hive2.BaseService;
+import org.apache.ambari.view.hive2.ConnectionFactory;
+import org.apache.ambari.view.hive2.ConnectionSystem;
+import org.apache.ambari.view.hive2.client.DDLDelegator;
+import org.apache.ambari.view.hive2.client.DDLDelegatorImpl;
+import org.apache.ambari.view.hive2.client.Row;
import org.apache.ambari.view.hive2.resources.jobs.viewJobs.Job;
import org.apache.ambari.view.hive2.resources.jobs.viewJobs.JobController;
import org.apache.ambari.view.hive2.resources.jobs.viewJobs.JobImpl;
@@ -45,6 +51,7 @@ import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.inject.Inject;
import javax.ws.rs.Consumes;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
@@ -57,6 +64,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -80,6 +89,9 @@ public class UploadService extends BaseService {
private final static Logger LOG =
LoggerFactory.getLogger(UploadService.class);
+ @Inject
+ protected ViewContext context;
+
private AmbariApi ambariApi;
protected JobResourceManager resourceManager;
@@ -348,22 +360,9 @@ public class UploadService extends BaseService {
private String uploadIntoTable(Reader reader, String databaseName, String tempTableName) {
try {
- String basePath = getHiveMetaStoreLocation();
-
- if (!basePath.endsWith("/")) {
- basePath = basePath + "/";
- }
-
- if (databaseName != null && !databaseName.equals(HIVE_DEFAULT_DB)) {
- basePath = basePath + databaseName + ".db/";
- }
-
- String fullPath = basePath + tempTableName + "/" + tempTableName + ".csv";
-
+ String fullPath = getHiveMetaStoreLocation(databaseName, tempTableName);
LOG.info("Uploading file into : {}", fullPath);
-
uploadFile(fullPath, new ReaderInputStream(reader));
-
return fullPath;
} catch (WebApplicationException e) {
LOG.error(getErrorMessage(e), e);
@@ -418,6 +417,40 @@ public class UploadService extends BaseService {
return job;
}
+ private String getHiveMetaStoreLocation(String db, String table) {
+ String locationColValue = "Location:";
+ String urlString = null;
+ DDLDelegator delegator = new DDLDelegatorImpl(context, ConnectionSystem.getInstance().getActorSystem(), ConnectionSystem.getInstance().getOperationController(context));
+ List<Row> result = delegator.getTableDescriptionFormatted(ConnectionFactory.create(context), db, table);
+ for (Row row : result) {
+ if (row != null && row.getRow().length > 1 && row.getRow()[0] != null && row.getRow()[0].toString().trim().equals(locationColValue)) {
+ urlString = row.getRow()[1] == null ? null : row.getRow()[1].toString();
+ break;
+ }
+ }
+
+ String tablePath = null;
+ if (null != urlString) {
+ try {
+ URI uri = new URI(urlString);
+ tablePath = uri.getPath();
+ } catch (URISyntaxException e) {
+ LOG.debug("Error occurred while parsing as url : ", urlString, e);
+ }
+ } else {
+ String basePath = getHiveMetaStoreLocation();
+ if (!basePath.endsWith("/")) {
+ basePath = basePath + "/";
+ }
+ if (db != null && !db.equals(HIVE_DEFAULT_DB)) {
+ basePath = basePath + db + ".db/";
+ }
+ tablePath = basePath + table;
+ }
+
+ return tablePath + "/" + table ;
+ }
+
private String getHiveMetaStoreLocation() {
String dir = context.getProperties().get(HIVE_METASTORE_LOCATION_KEY_VIEW_PROPERTY);
if (dir != null && !dir.trim().isEmpty()) {