You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/04/26 18:01:22 UTC
[16/34] ambari git commit: AMBARI-20814 : hive view 2.0 upload table
: handled partitions in the query (nitirajrathore)
AMBARI-20814 : hive view 2.0 upload table : handled partitions in the query (nitirajrathore)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1b7c023d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1b7c023d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1b7c023d
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 1b7c023d09827692c47d25bcdecb37669d342035
Parents: 64c9ef8
Author: Nitiraj Singh Rathore <ni...@gmail.com>
Authored: Tue Apr 25 17:15:39 2017 +0530
Committer: Nitiraj Singh Rathore <ni...@gmail.com>
Committed: Tue Apr 25 17:16:20 2017 +0530
----------------------------------------------------------------------
.../generators/InsertFromQueryGenerator.java | 41 +++++++++++++++++---
.../uploads/query/InsertFromQueryInput.java | 24 ++++++++----
.../databases/database/tables/upload-table.js | 16 +++++++-
3 files changed, 66 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
index 8e22fc7..19d4f06 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/InsertFromQueryGenerator.java
@@ -18,7 +18,10 @@
package org.apache.ambari.view.hive20.internal.query.generators;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
import org.apache.ambari.view.hive20.client.ColumnDescription;
import org.apache.ambari.view.hive20.exceptions.ServiceException;
import org.apache.ambari.view.hive20.internal.dto.ColumnInfo;
@@ -26,6 +29,10 @@ import org.apache.ambari.view.hive20.resources.uploads.query.InsertFromQueryInpu
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+import java.util.LinkedList;
+import java.util.List;
+
public class InsertFromQueryGenerator implements QueryGenerator{
protected final static Logger LOG =
LoggerFactory.getLogger(InsertFromQueryGenerator.class);
@@ -38,12 +45,34 @@ public class InsertFromQueryGenerator implements QueryGenerator{
@Override
public Optional<String> getQuery() throws ServiceException {
- StringBuilder insertQuery = new StringBuilder("INSERT INTO TABLE `").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
- .append("`").append(insertFromQueryInput.getToTable()).append("`")
- .append(" SELECT ");
+ StringBuilder insertQuery = new StringBuilder();
+ //Dynamic partition strict mode requires at least one static partition column. To turn this off set hive.exec.dynamic.partition.mode=nonstrict
+ insertQuery.append("set hive.exec.dynamic.partition.mode=nonstrict").append("\n");
+
+ insertQuery.append(" FROM ").append("`").append(insertFromQueryInput.getFromDatabase()).append("`.`")
+ .append(insertFromQueryInput.getFromTable()).append("` tempTable");
+
+ insertQuery.append(" INSERT INTO TABLE `").append(insertFromQueryInput.getToDatabase()).append('`').append(".")
+ .append("`").append(insertFromQueryInput.getToTable()).append("`");
+ // PARTITION (partcol1[=val1], partcol2[=val2] ...)
+ if(insertFromQueryInput.getPartitionedColumns() != null && insertFromQueryInput.getPartitionedColumns().size() > 0){
+ insertQuery.append(" PARTITION ").append("(");
+ insertQuery.append(Joiner.on(",").join(FluentIterable.from(insertFromQueryInput.getPartitionedColumns()).transform(new Function<ColumnInfo, String>() {
+ @Override
+ public String apply(ColumnInfo columnInfo) {
+ return "`" + columnInfo.getName() + "`";
+ }
+ })));
+ insertQuery.append(" ) ");
+ }
+
+ insertQuery.append(" SELECT ");
+ List<ColumnInfo> allColumns = new LinkedList<>(insertFromQueryInput.getNormalColumns());
+ // this order matters or first normal columns and in the last partitioned columns matters.
+ allColumns.addAll(insertFromQueryInput.getPartitionedColumns());
boolean first = true;
- for(ColumnInfo column : insertFromQueryInput.getHeader()){
+ for(ColumnInfo column : allColumns){
String type = column.getType();
boolean unhex = insertFromQueryInput.getUnhexInsert() && (
ColumnDescription.DataTypes.STRING.toString().equals(type)
@@ -59,6 +88,7 @@ public class InsertFromQueryGenerator implements QueryGenerator{
insertQuery.append("UNHEX(");
}
+ insertQuery.append("tempTable.");
insertQuery.append('`').append(column.getName()).append('`');
if(unhex) {
@@ -68,8 +98,7 @@ public class InsertFromQueryGenerator implements QueryGenerator{
first = false;
}
- insertQuery.append(" FROM ").append("`").append(insertFromQueryInput.getFromDatabase()).append(".")
- .append(insertFromQueryInput.getFromTable()).append("` ").append(";");
+ insertQuery.append(";");
String query = insertQuery.toString();
LOG.info("Insert From Query : {}", query);
return Optional.of(query);
http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java
index 4ff61b4..b74ba9b 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/uploads/query/InsertFromQueryInput.java
@@ -28,28 +28,38 @@ public class InsertFromQueryInput {
private String fromTable;
private String toDatabase;
private String toTable;
- private List<ColumnInfo> header;
+ private List<ColumnInfo> partitionedColumns;
+ private List<ColumnInfo> normalColumns;
private Boolean unhexInsert = Boolean.FALSE;
public InsertFromQueryInput() {
}
public InsertFromQueryInput(String fromDatabase, String fromTable, String toDatabase, String toTable,
- List<ColumnInfo> header, Boolean unhexInsert) {
+ List<ColumnInfo> partitionedColumns, List<ColumnInfo> normalColumns, Boolean unhexInsert) {
this.fromDatabase = fromDatabase;
this.fromTable = fromTable;
this.toDatabase = toDatabase;
this.toTable = toTable;
- this.header = header;
+ this.partitionedColumns = partitionedColumns;
+ this.normalColumns = normalColumns;
this.unhexInsert = unhexInsert;
}
- public List<ColumnInfo> getHeader() {
- return header;
+ public List<ColumnInfo> getPartitionedColumns() {
+ return partitionedColumns;
}
- public void setHeader(List<ColumnInfo> header) {
- this.header = header;
+ public void setPartitionedColumns(List<ColumnInfo> partitionedColumns) {
+ this.partitionedColumns = partitionedColumns;
+ }
+
+ public List<ColumnInfo> getNormalColumns() {
+ return normalColumns;
+ }
+
+ public void setNormalColumns(List<ColumnInfo> normalColumns) {
+ this.normalColumns = normalColumns;
}
public Boolean getUnhexInsert() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/1b7c023d/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js
index 5fee140..f47d820 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/routes/databases/database/tables/upload-table.js
@@ -279,6 +279,7 @@ export default NewTable.extend(UILoggerMixin, {
column.set("scale");
column.set("precision");
}
+ column.set("isPartitioned", false); // remove all partitioning information from temp table.
return column;
});
@@ -421,7 +422,17 @@ export default NewTable.extend(UILoggerMixin, {
console.log("insertIntoTable");
this.pushUploadProgressInfos(this.formatMessage('hive.messages.startingToInsertRows'));
- let headers = tableData.get("tableMeta").columns.map(function(column){
+ let partitionedColumns = tableData.get("tableMeta").columns.filter(function(column){
+ return column.isPartitioned;
+ }).map(function(column){
+ var header = JSON.parse(JSON.stringify(column));
+ header.type = column.type.label;
+ return header;
+ });
+
+ let normalColumns = tableData.get("tableMeta").columns.filter(function(column){
+ return !column.isPartitioned;
+ }).map(function(column){
var header = JSON.parse(JSON.stringify(column));
header.type = column.type.label;
return header;
@@ -432,7 +443,8 @@ export default NewTable.extend(UILoggerMixin, {
"fromTable": tableData.get("tempTableMeta").name,
"toDatabase": tableData.get("database"),
"toTable": tableData.get("tableMeta").name,
- "header": headers,
+ "partitionedColumns": partitionedColumns,
+ "normalColumns": normalColumns,
"unhexInsert": tableData.fileFormatInfo.containsEndlines
});
},