You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/03/01 20:39:02 UTC
[45/50] ambari git commit: AMBARI-20113 : hive20 view : fixed : Not
able to compute the table statistics for partitioned table (nitirajrathore)
AMBARI-20113 : hive20 view : fixed : Not able to compute the table statistics for partitioned table (nitirajrathore)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a749f97a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a749f97a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a749f97a
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: a749f97abbc44217fbb6f8e4868deaf6c4811fab
Parents: 8bcc953
Author: Nitiraj Singh Rathore <ni...@gmail.com>
Authored: Wed Mar 1 23:30:35 2017 +0530
Committer: Nitiraj Singh Rathore <ni...@gmail.com>
Committed: Wed Mar 1 23:30:35 2017 +0530
----------------------------------------------------------------------
.../generators/AnalyzeTableQueryGenerator.java | 49 ++++-
.../view/hive20/resources/browser/DDLProxy.java | 6 +-
.../hive20/resources/browser/DDLService.java | 4 +-
.../AnalyzeTableQueryGeneratorSpecTest.groovy | 194 +++++++++++++++++++
4 files changed, 242 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
index 1d68407..dbe9d43 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
@@ -18,23 +18,58 @@
package org.apache.ambari.view.hive20.internal.query.generators;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
import org.apache.ambari.view.hive20.exceptions.ServiceException;
+import org.apache.ambari.view.hive20.internal.dto.ColumnInfo;
+import org.apache.ambari.view.hive20.internal.dto.TableMeta;
+
+import javax.annotation.Nullable;
+
+import static org.apache.ambari.view.hive20.internal.query.generators.QueryGenerationUtils.isNullOrEmpty;
public class AnalyzeTableQueryGenerator implements QueryGenerator {
- private final String databaseName;
- private final String tableName;
+ private TableMeta tableMeta;
private final Boolean shouldAnalyzeColumns;
- public AnalyzeTableQueryGenerator(String databaseName, String tableName, Boolean shouldAnalyzeColumns) {
- this.databaseName = databaseName;
- this.tableName = tableName;
+ public AnalyzeTableQueryGenerator(TableMeta tableMeta, Boolean shouldAnalyzeColumns) {
+ this.tableMeta = tableMeta;
this.shouldAnalyzeColumns = shouldAnalyzeColumns;
}
@Override
public Optional<String> getQuery() throws ServiceException {
- return Optional.of("ANALYZE TABLE " + "`" + databaseName + "`.`" + tableName + "`" + " COMPUTE STATISTICS " +
- (shouldAnalyzeColumns? " FOR COLUMNS ": "") + ";");
+ StringBuilder query = new StringBuilder("ANALYZE TABLE " );
+ query.append("`").append(tableMeta.getDatabase()).append("`").append(".").append("`").append(tableMeta.getTable()).append("`");
+
+ if( null != tableMeta.getPartitionInfo() && !isNullOrEmpty(tableMeta.getPartitionInfo().getColumns())){
+ query.append(" PARTITION (")
+ .append(Joiner.on(",")
+ .join(FluentIterable.from(tableMeta.getPartitionInfo().getColumns())
+ .transform(
+ new Function<ColumnInfo, Object>() {
+ @Nullable
+ @Override
+ public Object apply(@Nullable ColumnInfo columnInfo) {
+ return columnInfo.getName();
+ }
+ })
+ )
+ )
+ .append(")");
+ }
+
+
+ query.append(" COMPUTE STATISTICS ");
+
+ if(shouldAnalyzeColumns){
+ query.append(" FOR COLUMNS ");
+ }
+
+ query.append(";");
+
+ return Optional.of(query.toString());
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
index e433dc4..0c93ba3 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
@@ -332,8 +332,10 @@ public class DDLProxy {
}
}
- public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns, JobResourceManager resourceManager) throws ServiceException {
- AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(databaseName, tableName, shouldAnalyzeColumns);
+ public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns, JobResourceManager resourceManager, ConnectionConfig hiveConnectionConfig) throws ServiceException {
+ TableMeta tableMeta = this.getTableProperties(context, hiveConnectionConfig, databaseName, tableName);
+
+ AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(tableMeta, shouldAnalyzeColumns);
Optional<String> analyzeTable = queryGenerator.getQuery();
String jobTitle = "Analyze table " + databaseName + "." + tableName;
if(analyzeTable.isPresent()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
index 89b9d84..f5a4781 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
-import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
@@ -184,7 +183,8 @@ public class DDLService extends BaseService {
shouldAnalyzeColumns = Boolean.valueOf(analyzeColumns.trim());
}
try {
- Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager());
+ ConnectionConfig hiveConnectionConfig = getHiveConnectionConfig();
+ Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager(), hiveConnectionConfig);
JSONObject response = new JSONObject();
response.put("job", job);
return Response.status(Response.Status.ACCEPTED).entity(response).build();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy
new file mode 100644
index 0000000..7bc5850
--- /dev/null
+++ b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy
@@ -0,0 +1,194 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ambari.view.hive20.internal.query.generators;
+
+import com.google.gson.Gson;
+import org.apache.ambari.view.hive20.internal.dto.TableMeta;
+import spock.lang.Specification;
+import com.google.common.base.Optional;
+
+class AnalyzeTableQueryGeneratorSpecTest extends Specification {
+ def "analyze with partition and for columns"() {
+ setup:
+ String tableMetaJson = "{" +
+ "\"database\": \"d1\"," +
+ "\"table\": \"t2\"," +
+ "\"columns\": [{" +
+ "\"name\": \"col_name1\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name1 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name2\"," +
+ "\"type\": \"decimal(10,2)\"," +
+ "\"comment\": \"col_name2 comment\"" +
+ "}]," +
+ "\"partitionInfo\": {" +
+ "\"columns\": [{" +
+ "\"name\": \"col_name4\"," +
+ "\"type\": \"char(1)\"," +
+ "\"comment\": \"col_name4 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name3\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name3 comment\"" +
+ "}]" +
+ "}" +
+ "}";
+
+ TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+ AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true);
+
+ when:
+ Optional<String> databaseDeleteQuery = generator.getQuery()
+
+ then:
+ databaseDeleteQuery.isPresent()
+
+ when:
+ String query = databaseDeleteQuery.get();
+
+ then:
+ query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS FOR COLUMNS ;"
+ }
+ def "analyze with partition"() {
+ setup:
+ String tableMetaJson = "{" +
+ "\"database\": \"d1\"," +
+ "\"table\": \"t2\"," +
+ "\"columns\": [{" +
+ "\"name\": \"col_name1\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name1 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name2\"," +
+ "\"type\": \"decimal(10,2)\"," +
+ "\"comment\": \"col_name2 comment\"" +
+ "}]," +
+ "\"partitionInfo\": {" +
+ "\"columns\": [{" +
+ "\"name\": \"col_name4\"," +
+ "\"type\": \"char(1)\"," +
+ "\"comment\": \"col_name4 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name3\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name3 comment\"" +
+ "}]" +
+ "}" +
+ "}";
+
+ TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+ AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false);
+
+ when:
+ Optional<String> databaseDeleteQuery = generator.getQuery()
+
+ then:
+ databaseDeleteQuery.isPresent()
+
+ when:
+ String query = databaseDeleteQuery.get();
+
+ then:
+ query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS ;"
+ }
+
+ def "analyze without partition"() {
+ setup:
+ String tableMetaJson = "{" +
+ "\"database\": \"d1\"," +
+ "\"table\": \"t2\"," +
+ "\"columns\": [{" +
+ "\"name\": \"col_name1\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name1 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name2\"," +
+ "\"type\": \"decimal(10,2)\"," +
+ "\"comment\": \"col_name2 comment\"" +
+ "}," +
+ "{" +
+ "\"name\": \"col_name4\"," +
+ "\"type\": \"char(1)\"," +
+ "\"comment\": \"col_name4 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name3\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name3 comment\"" +
+ "}" +
+ "]" +
+ "}";
+
+ TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+ AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true);
+
+ when:
+ Optional<String> databaseDeleteQuery = generator.getQuery()
+
+ then:
+ databaseDeleteQuery.isPresent()
+
+ when:
+ String query = databaseDeleteQuery.get();
+
+ then:
+ query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS FOR COLUMNS ;"
+ }
+
+ def "analyze for table only"() {
+ setup:
+ String tableMetaJson = "{" +
+ "\"database\": \"d1\"," +
+ "\"table\": \"t2\"," +
+ "\"columns\": [{" +
+ "\"name\": \"col_name1\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name1 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name2\"," +
+ "\"type\": \"decimal(10,2)\"," +
+ "\"comment\": \"col_name2 comment\"" +
+ "}," +
+ "{" +
+ "\"name\": \"col_name4\"," +
+ "\"type\": \"char(1)\"," +
+ "\"comment\": \"col_name4 comment\"" +
+ "}, {" +
+ "\"name\": \"col_name3\"," +
+ "\"type\": \"string\"," +
+ "\"comment\": \"col_name3 comment\"" +
+ "}" +
+ "]" +
+ "}";
+
+ TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+ AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false);
+
+ when:
+ Optional<String> databaseDeleteQuery = generator.getQuery()
+
+ then:
+ databaseDeleteQuery.isPresent()
+
+ when:
+ String query = databaseDeleteQuery.get();
+
+ then:
+ query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS ;"
+ }
+}