You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hj...@apache.org on 2014/12/05 09:21:24 UTC
[20/29] tajo git commit: TAJO-1194: 'INSERT OVERWRITE .. SELECT' does
not remove existing data when result is empty. (missing code)
TAJO-1194: 'INSERT OVERWRITE .. SELECT' does not remove existing data when result is empty. (missing code)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2a69bcc2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2a69bcc2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2a69bcc2
Branch: refs/heads/hbase_storage
Commit: 2a69bcc218a53eced0cdb9217de1fdd1837c0827
Parents: f6da07b
Author: JaeHwa Jung <bl...@apache.org>
Authored: Thu Dec 4 18:54:03 2014 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Thu Dec 4 18:54:03 2014 +0900
----------------------------------------------------------------------
.../tajo/engine/query/TestTablePartitions.java | 232 ++++++++++++-------
1 file changed, 142 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2a69bcc2/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
index 15cbde0..b1e1bec 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
@@ -19,10 +19,7 @@
package org.apache.tajo.engine.query;
import com.google.common.collect.Maps;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.DeflateCodec;
@@ -44,6 +41,7 @@ import org.apache.tajo.jdbc.TajoResultSet;
import org.apache.tajo.master.querymaster.QueryMasterTask;
import org.apache.tajo.plan.logical.NodeType;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.worker.TajoWorker;
import org.junit.Test;
@@ -70,7 +68,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testCreateColumnPartitionedTable() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testCreateColumnPartitionedTable");
ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ");
+ "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
@@ -78,8 +76,8 @@ public class TestTablePartitions extends QueryTestCaseBase {
assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
res = testBase.execute(
- "insert overwrite into " + tableName + " select l_orderkey, l_partkey, " +
- "l_quantity from lineitem");
+ "insert overwrite into " + tableName + " select l_orderkey, l_partkey, " +
+ "l_quantity from lineitem");
MasterPlan plan = getQueryPlan(res);
ExecutionBlock rootEB = plan.getRoot();
@@ -113,7 +111,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testCreateColumnPartitionedTableWithJoin() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testCreateColumnPartitionedTableWithJoin");
ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ");
+ "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
@@ -121,8 +119,8 @@ public class TestTablePartitions extends QueryTestCaseBase {
assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
res = testBase.execute(
- "insert overwrite into " + tableName + " select l_orderkey, l_partkey, " +
- "l_quantity from lineitem join orders on l_orderkey = o_orderkey");
+ "insert overwrite into " + tableName + " select l_orderkey, l_partkey, " +
+ "l_quantity from lineitem join orders on l_orderkey = o_orderkey");
MasterPlan plan = getQueryPlan(res);
ExecutionBlock rootEB = plan.getRoot();
@@ -158,7 +156,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testCreateColumnPartitionedTableWithSelectedColumns() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testCreateColumnPartitionedTableWithSelectedColumns");
ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
+ "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
@@ -166,7 +164,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
assertEquals(4, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
res = executeString("insert overwrite into " + tableName + " (col1, col2, key) select l_orderkey, " +
- "l_partkey, l_quantity from lineitem");
+ "l_partkey, l_quantity from lineitem");
res.close();
}
@@ -174,20 +172,20 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableByOneColumn() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByOneColumn");
ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
+ "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString("insert overwrite into " + tableName
- + " (col1, col2, key) select l_orderkey, l_partkey, l_quantity from lineitem");
+ + " (col1, col2, key) select l_orderkey, l_partkey, l_quantity from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
assertPartitionDirectories(desc);
res = executeString(
- "select distinct * from " + tableName + " where (key = 45.0 or key = 38.0) and null_col is null");
+ "select distinct * from " + tableName + " where (key = 45.0 or key = 38.0) and null_col is null");
Map<Double, int []> resultRows1 = Maps.newHashMap();
resultRows1.put(45.0d, new int[]{3, 2});
@@ -219,14 +217,14 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testQueryCasesOnColumnPartitionedTable() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testQueryCasesOnColumnPartitionedTable");
ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
+ "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString(
- "insert overwrite into " + tableName
- + " (col1, col2, key) select l_orderkey, l_partkey, l_quantity from lineitem");
+ "insert overwrite into " + tableName
+ + " (col1, col2, key) select l_orderkey, l_partkey, l_quantity from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
@@ -299,14 +297,14 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableByThreeColumns() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByThreeColumns");
ResultSet res = testBase.execute(
- "create table " + tableName + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) ");
+ "create table " + tableName + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) ");
res.close();
TajoTestingCluster cluster = testBase.getTestingCluster();
CatalogService catalog = cluster.getMaster().getCatalog();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString("insert overwrite into " + tableName
- + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
+ + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
@@ -363,14 +361,14 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testInsertIntoColumnPartitionedTableByThreeColumns");
ResultSet res = testBase.execute(
- "create table " + tableName + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) ");
+ "create table " + tableName + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) ");
res.close();
TajoTestingCluster cluster = testBase.getTestingCluster();
CatalogService catalog = cluster.getMaster().getCatalog();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString("insert into " + tableName
- + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
+ + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
@@ -423,7 +421,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
// insert into already exists partitioned table
res = executeString("insert into " + tableName
- + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
+ + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
res.close();
desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
@@ -446,15 +444,15 @@ public class TestTablePartitions extends QueryTestCaseBase {
assertEquals(5, desc.getStats().getNumRows().intValue());
}
String expected = "N\n" +
- "N\n" +
- "N\n" +
- "N\n" +
- "N\n" +
- "N\n" +
- "R\n" +
- "R\n" +
- "R\n" +
- "R\n";
+ "N\n" +
+ "N\n" +
+ "N\n" +
+ "N\n" +
+ "N\n" +
+ "R\n" +
+ "R\n" +
+ "R\n" +
+ "R\n";
String tableData = getTableFileContents(new Path(desc.getPath()));
assertEquals(expected, tableData);
@@ -463,30 +461,30 @@ public class TestTablePartitions extends QueryTestCaseBase {
String resultSetData = resultSetToString(res);
res.close();
expected = "col4,col1,col2,col3\n" +
- "-------------------------------\n" +
- "N,2,2,38.0\n" +
- "N,2,2,38.0\n" +
- "R,3,2,45.0\n" +
- "R,3,2,45.0\n";
+ "-------------------------------\n" +
+ "N,2,2,38.0\n" +
+ "N,2,2,38.0\n" +
+ "R,3,2,45.0\n" +
+ "R,3,2,45.0\n";
assertEquals(expected, resultSetData);
res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2");
resultSetData = resultSetToString(res);
res.close();
expected = "col4,col1,col2,col3\n" +
- "-------------------------------\n" +
- "N,2,2,38.0\n" +
- "N,2,2,38.0\n" +
- "R,3,2,45.0\n" +
- "R,3,2,45.0\n" +
- "R,3,3,49.0\n" +
- "R,3,3,49.0\n";
+ "-------------------------------\n" +
+ "N,2,2,38.0\n" +
+ "N,2,2,38.0\n" +
+ "R,3,2,45.0\n" +
+ "R,3,2,45.0\n" +
+ "R,3,3,49.0\n" +
+ "R,3,3,49.0\n";
assertEquals(expected, resultSetData);
// Check not to remove existing partition directories.
res = executeString("insert overwrite into " + tableName
- + " select l_returnflag, l_orderkey, l_partkey, 30.0 as l_quantity from lineitem "
- + " where l_orderkey = 1 and l_partkey = 1 and l_linenumber = 1");
+ + " select l_returnflag, l_orderkey, l_partkey, 30.0 as l_quantity from lineitem "
+ + " where l_orderkey = 1 and l_partkey = 1 and l_linenumber = 1");
res.close();
desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
@@ -513,12 +511,12 @@ public class TestTablePartitions extends QueryTestCaseBase {
resultSetData = resultSetToString(res);
res.close();
expected = "col4,col1,col2,col3\n" +
- "-------------------------------\n" +
- "N,1,1,17.0\n" +
- "N,1,1,17.0\n" +
- "N,1,1,30.0\n" +
- "N,1,1,36.0\n" +
- "N,1,1,36.0\n";
+ "-------------------------------\n" +
+ "N,1,1,17.0\n" +
+ "N,1,1,17.0\n" +
+ "N,1,1,30.0\n" +
+ "N,1,1,36.0\n" +
+ "N,1,1,36.0\n";
assertEquals(expected, resultSetData);
@@ -541,14 +539,14 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableByOneColumnsWithCompression() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByOneColumnsWithCompression");
ResultSet res = executeString(
- "create table " + tableName + " (col2 int4, col3 float8) USING csv " +
- "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
- "PARTITION BY column(col1 int4)");
+ "create table " + tableName + " (col2 int4, col3 float8) USING csv " +
+ "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
+ "PARTITION BY column(col1 int4)");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString(
- "insert overwrite into " + tableName + " select l_partkey, l_quantity, l_orderkey from lineitem");
+ "insert overwrite into " + tableName + " select l_partkey, l_quantity, l_orderkey from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
if (!testingCluster.isHCatalogStoreRunning()) {
@@ -577,15 +575,15 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByTwoColumnsWithCompression");
ResultSet res = executeString("create table " + tableName + " (col3 float8, col4 text) USING csv " +
- "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
- "PARTITION by column(col1 int4, col2 int4)");
+ "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
+ "PARTITION by column(col1 int4, col2 int4)");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString(
- "insert overwrite into " + tableName +
- " select l_quantity, l_returnflag, l_orderkey, l_partkey from lineitem");
+ "insert overwrite into " + tableName +
+ " select l_quantity, l_returnflag, l_orderkey, l_partkey from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
if (!testingCluster.isHCatalogStoreRunning()) {
@@ -621,16 +619,16 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableByThreeColumnsWithCompression() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByThreeColumnsWithCompression");
ResultSet res = executeString(
- "create table " + tableName + " (col4 text) USING csv " +
- "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
- "partition by column(col1 int4, col2 int4, col3 float8)");
+ "create table " + tableName + " (col4 text) USING csv " +
+ "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
+ "partition by column(col1 int4, col2 int4, col3 float8)");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString(
- "insert overwrite into " + tableName +
- " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
+ "insert overwrite into " + tableName +
+ " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
if (!testingCluster.isHCatalogStoreRunning()) {
@@ -704,16 +702,16 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableNoMatchedPartition() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableNoMatchedPartition");
ResultSet res = executeString(
- "create table " + tableName + " (col4 text) USING csv " +
- "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
- "partition by column(col1 int4, col2 int4, col3 float8)");
+ "create table " + tableName + " (col4 text) USING csv " +
+ "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') " +
+ "partition by column(col1 int4, col2 int4, col3 float8)");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
res = executeString(
- "insert overwrite into " + tableName +
- " select l_returnflag , l_orderkey, l_partkey, l_quantity from lineitem");
+ "insert overwrite into " + tableName +
+ " select l_returnflag , l_orderkey, l_partkey, l_quantity from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
if (!testingCluster.isHCatalogStoreRunning()) {
@@ -760,13 +758,13 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableWithSmallerExpressions1() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions1");
ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
+ "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
ClientProtos.SubmitQueryResponse response = client.executeQuery("insert overwrite into " + tableName
- + " select l_orderkey, l_partkey from lineitem");
+ + " select l_orderkey, l_partkey from lineitem");
assertTrue(response.hasErrorMessage());
assertEquals(response.getErrorMessage(), "INSERT has smaller expressions than target columns\n");
@@ -780,13 +778,13 @@ public class TestTablePartitions extends QueryTestCaseBase {
public final void testColumnPartitionedTableWithSmallerExpressions2() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions2");
ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
+ "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) ");
res.close();
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
ClientProtos.SubmitQueryResponse response = client.executeQuery("insert overwrite into " + tableName
- + " select l_returnflag , l_orderkey, l_partkey from lineitem");
+ + " select l_returnflag , l_orderkey, l_partkey from lineitem");
assertTrue(response.hasErrorMessage());
assertEquals(response.getErrorMessage(), "INSERT has smaller expressions than target columns\n");
@@ -805,11 +803,11 @@ public class TestTablePartitions extends QueryTestCaseBase {
res.close();
res = executeString("create table testinsertquery1.table1 " +
- "(col1 int4, col2 int4, col3 float8)");
+ "(col1 int4, col2 int4, col3 float8)");
res.close();
res = executeString("create table testinsertquery2.table1 " +
- "(col1 int4, col2 int4, col3 float8)");
+ "(col1 int4, col2 int4, col3 float8)");
res.close();
CatalogService catalog = testingCluster.getMaster().getCatalog();
@@ -817,7 +815,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
assertTrue(catalog.existsTable("testinsertquery2", "table1"));
res = executeString("insert overwrite into testinsertquery1.table1 " +
- "select l_orderkey, l_partkey, l_quantity from default.lineitem;");
+ "select l_orderkey, l_partkey, l_quantity from default.lineitem;");
res.close();
TableDesc desc = catalog.getTableDesc("testinsertquery1", "table1");
@@ -826,7 +824,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
}
res = executeString("insert overwrite into testinsertquery2.table1 " +
- "select col1, col2, col3 from testinsertquery1.table1;");
+ "select col1, col2, col3 from testinsertquery1.table1;");
res.close();
desc = catalog.getTableDesc("testinsertquery2", "table1");
@@ -835,6 +833,39 @@ public class TestTablePartitions extends QueryTestCaseBase {
}
}
+ @Test
+ public final void testColumnPartitionedTableWithSmallerExpressions5() throws Exception {
+ String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions5");
+ ResultSet res = executeString(
+ "create table " + tableName + " (col1 text) partition by column(col2 text) ");
+ res.close();
+
+ assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
+
+ res = executeString("insert overwrite into " + tableName + "(col1) select l_returnflag from lineitem");
+ res.close();
+ res = executeString("select * from " + tableName);
+ assertResultSet(res);
+ res.close();
+ }
+
+ @Test
+ public final void testColumnPartitionedTableWithSmallerExpressions6() throws Exception {
+ String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions6");
+ ResultSet res = executeString(
+ "create table " + tableName + " (col1 text) partition by column(col2 text) ");
+ res.close();
+
+ assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
+
+ res = executeString(
+ "insert overwrite into " + tableName + "(col1) select l_returnflag from lineitem where l_orderkey = 1");
+ res.close();
+ res = executeString("select * from " + tableName);
+ assertResultSet(res);
+ res.close();
+ }
+
private MasterPlan getQueryPlan(ResultSet res) {
QueryId queryId = ((TajoResultSet)res).getQueryId();
for (TajoWorker eachWorker: testingCluster.getTajoWorkers()) {
@@ -865,8 +896,8 @@ public class TestTablePartitions extends QueryTestCaseBase {
int totalBytes = 0;
Random rand = new Random(System.currentTimeMillis());
String col2Data = "Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2" +
- "Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2" +
- "Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2";
+ "Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2" +
+ "Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2";
int index = 0;
while(true) {
@@ -901,9 +932,9 @@ public class TestTablePartitions extends QueryTestCaseBase {
} finally {
testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.varname,
- TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.defaultVal);
+ TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.defaultVal);
testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME.varname,
- TajoConf.ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME.defaultVal);
+ TajoConf.ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME.defaultVal);
executeString("DROP TABLE test_partition PURGE").close();
executeString("DROP TABLE testScatteredHashShuffle PURGE").close();
}
@@ -916,17 +947,17 @@ public class TestTablePartitions extends QueryTestCaseBase {
executeDDL("lineitemspecial_ddl.sql", "lineitemspecial.tbl");
executeString("CREATE TABLE IF NOT EXISTS pTable947 (id int, name text) PARTITION BY COLUMN (type text)")
- .close();
+ .close();
executeString("INSERT OVERWRITE INTO pTable947 SELECT l_orderkey, l_shipinstruct, l_shipmode FROM lineitemspecial")
- .close();
+ .close();
ResultSet res = executeString("select * from pTable947 where type='RA:*?><I/L#%S' or type='AIR'");
String resStr = resultSetToString(res);
String expected =
- "id,name,type\n" +
- "-------------------------------\n"
- + "3,NONE,AIR\n"
- + "3,TEST SPECIAL CHARS,RA:*?><I/L#%S\n";
+ "id,name,type\n" +
+ "-------------------------------\n"
+ + "3,NONE,AIR\n"
+ + "3,TEST SPECIAL CHARS,RA:*?><I/L#%S\n";
assertEquals(expected, resStr);
cleanupQuery(res);
@@ -939,9 +970,9 @@ public class TestTablePartitions extends QueryTestCaseBase {
executeDDL("lineitemspecial_ddl.sql", "lineitemspecial.tbl");
executeString("CREATE TABLE IF NOT EXISTS pTable948 (id int, name text) PARTITION BY COLUMN (type text)")
- .close();
+ .close();
executeString("INSERT OVERWRITE INTO pTable948 SELECT l_orderkey, l_shipinstruct, l_shipmode FROM lineitemspecial")
- .close();
+ .close();
ResultSet res = executeString("select * from pTable948 where type='RA:*?><I/L#%S'");
assertResultSet(res);
@@ -951,4 +982,25 @@ public class TestTablePartitions extends QueryTestCaseBase {
assertResultSet(res);
cleanupQuery(res);
}
+
+ @Test
+ public final void testIgnoreFilesInIntermediateDir() throws Exception {
+ // See - TAJO-1219: Files located in intermediate directories of partitioned table should be ignored
+ // It verifies that Tajo ignores files located in intermediate directories of partitioned table.
+
+ Path testDir = CommonTestingUtil.getTestDir();
+
+ executeString(
+ "CREATE EXTERNAL TABLE testIgnoreFilesInIntermediateDir (col1 int) USING CSV PARTITION BY COLUMN (col2 text) " +
+ "LOCATION '" + testDir + "'");
+
+ FileSystem fs = testDir.getFileSystem(conf);
+ FSDataOutputStream fos = fs.create(new Path(testDir, "table1.data"));
+ fos.write("a|b|c".getBytes());
+ fos.close();
+
+ ResultSet res = executeString("select * from testIgnoreFilesInIntermediateDir;");
+ assertFalse(res.next());
+ res.close();
+ }
}