You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/09/20 06:05:06 UTC
svn commit: r1387850 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
test/queries/clientpositive/ctas_uses_database_location.q
test/results/clientpositive/ctas_uses_database_location.q.out
Author: namit
Date: Thu Sep 20 04:05:06 2012
New Revision: 1387850
URL: http://svn.apache.org/viewvc?rev=1387850&view=rev
Log:
HIVE-3486 CTAS in database with location on non-default name node fails
(Kevin Wilfong via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/ctas_uses_database_location.q
hive/trunk/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1387850&r1=1387849&r2=1387850&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Sep 20 04:05:06 2012
@@ -70,8 +70,8 @@ import org.apache.hadoop.hive.ql.exec.Re
import org.apache.hadoop.hive.ql.exec.RecordWriter;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
-import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -1054,13 +1054,21 @@ public class SemanticAnalyzer extends Ba
ctx.setResFile(null);
// allocate a temporary output dir on the location of the table
- String location = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
+ String tableName = getUnescapedName((ASTNode)ast.getChild(0));
+ Table newTable = db.newTable(tableName);
+ Path location;
+ try {
+ Warehouse wh = new Warehouse(conf);
+ location = wh.getDatabasePath(db.getDatabase(newTable.getDbName()));
+ } catch (MetaException e) {
+ throw new SemanticException(e);
+ }
try {
fname = ctx.getExternalTmpFileURI(
- FileUtils.makeQualified(new Path(location), conf).toUri());
+ FileUtils.makeQualified(location, conf).toUri());
} catch (Exception e) {
throw new SemanticException(generateErrorMessage(ast,
- "Error creating temporary folder on: " + location), e);
+ "Error creating temporary folder on: " + location.toString()), e);
}
if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
tableSpec ts = new tableSpec(db, conf, this.ast);
Added: hive/trunk/ql/src/test/queries/clientpositive/ctas_uses_database_location.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ctas_uses_database_location.q?rev=1387850&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ctas_uses_database_location.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/ctas_uses_database_location.q Thu Sep 20 04:05:06 2012
@@ -0,0 +1,14 @@
+set hive.metastore.warehouse.dir=invalid_scheme://${system:test.tmp.dir};
+
+-- Tests that CTAS queries in non-default databases use the location of the database
+-- not the hive.metastore.warehouse.dir for intermediate files (FileSinkOperator output).
+-- If hive.metastore.warehouse.dir were used this would fail because the scheme is invalid.
+
+CREATE DATABASE db1
+LOCATION 'pfile://${system:test.tmp.dir}/db1';
+
+USE db1;
+EXPLAIN CREATE TABLE table_db1 AS SELECT * FROM default.src;
+CREATE TABLE table_db1 AS SELECT * FROM default.src;
+
+DESCRIBE FORMATTED table_db1;
Added: hive/trunk/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out?rev=1387850&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out Thu Sep 20 04:05:06 2012
@@ -0,0 +1,159 @@
+PREHOOK: query: -- Tests that CTAS queries in non-default databases use the location of the database
+-- not the hive.metastore.warehouse.dir for intermediate files (FileSinkOperator output).
+-- If hive.metastore.warehouse.dir were used this would fail because the scheme is invalid.
+
+CREATE DATABASE db1
+#### A masked pattern was here ####
+PREHOOK: type: CREATEDATABASE
+POSTHOOK: query: -- Tests that CTAS queries in non-default databases use the location of the database
+-- not the hive.metastore.warehouse.dir for intermediate files (FileSinkOperator output).
+-- If hive.metastore.warehouse.dir were used this would fail because the scheme is invalid.
+
+CREATE DATABASE db1
+#### A masked pattern was here ####
+POSTHOOK: type: CREATEDATABASE
+PREHOOK: query: USE db1
+PREHOOK: type: SWITCHDATABASE
+POSTHOOK: query: USE db1
+POSTHOOK: type: SWITCHDATABASE
+PREHOOK: query: EXPLAIN CREATE TABLE table_db1 AS SELECT * FROM default.src
+PREHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: query: EXPLAIN CREATE TABLE table_db1 AS SELECT * FROM default.src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+ABSTRACT SYNTAX TREE:
+ (TOK_CREATETABLE (TOK_TABNAME table_db1) TOK_LIKETABLE (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME default src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-8 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-8
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ name: db1.table_db1
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-8
+ Create Table Operator:
+ Create Table
+ columns: key string, value string
+ if not exists: false
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ # buckets: -1
+ output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+ name: table_db1
+ isExternal: false
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ name: db1.table_db1
+
+ Stage: Stage-5
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ name: db1.table_db1
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+
+PREHOOK: query: CREATE TABLE table_db1 AS SELECT * FROM default.src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+POSTHOOK: query: CREATE TABLE table_db1 AS SELECT * FROM default.src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: db1@table_db1
+PREHOOK: query: DESCRIBE FORMATTED table_db1
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED table_db1
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+key string None
+value string None
+
+# Detailed Table Information
+Database: db1
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ numFiles 1
+ numPartitions 0
+ numRows 500
+ rawDataSize 5312
+ totalSize 5812
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1