You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/05/12 05:14:00 UTC
[01/14] tajo git commit: initial work for TAJO-1591.
Repository: tajo
Updated Branches:
refs/heads/index_support 410ca3188 -> 6f4409629
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 456ea00..7c75973 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -108,14 +108,14 @@ public class TestStorages {
" ]\n" +
"}\n";
- private StoreType storeType;
+ private String storeType;
private boolean splitable;
private boolean statsable;
private boolean seekable;
private Path testDir;
private FileSystem fs;
- public TestStorages(StoreType type, boolean splitable, boolean statsable, boolean seekable) throws IOException {
+ public TestStorages(String type, boolean splitable, boolean statsable, boolean seekable) throws IOException {
this.storeType = type;
this.splitable = splitable;
this.statsable = statsable;
@@ -123,7 +123,7 @@ public class TestStorages {
conf = new TajoConf();
- if (storeType == StoreType.RCFILE) {
+ if (storeType.equalsIgnoreCase("RCFILE")) {
conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
}
@@ -135,14 +135,14 @@ public class TestStorages {
public static Collection<Object[]> generateParameters() {
return Arrays.asList(new Object[][] {
//type, splitable, statsable, seekable
- {StoreType.CSV, true, true, true},
- {StoreType.RAW, false, true, true},
- {StoreType.RCFILE, true, true, false},
- {StoreType.PARQUET, false, false, false},
- {StoreType.SEQUENCEFILE, true, true, false},
- {StoreType.AVRO, false, false, false},
- {StoreType.TEXTFILE, true, true, true},
- {StoreType.JSON, true, true, false},
+ {"CSV", true, true, true},
+ {"RAW", false, true, true},
+ {"RCFILE", true, true, false},
+ {"PARQUET", false, false, false},
+ {"SEQUENCEFILE", true, true, false},
+ {"AVRO", false, false, false},
+ {"TEXT", true, true, true},
+ {"JSON", true, true, false},
});
}
@@ -203,7 +203,7 @@ public class TestStorages {
@Test
public void testRCFileSplitable() throws IOException {
- if (storeType == StoreType.RCFILE) {
+ if (storeType.equalsIgnoreCase("StoreType.RCFILE")) {
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
@@ -265,7 +265,7 @@ public class TestStorages {
TableMeta meta = CatalogUtil.newTableMeta(storeType);
meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
- if (storeType == StoreType.AVRO) {
+ if (storeType.equalsIgnoreCase("AVRO")) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_PROJECTION_AVRO_SCHEMA);
}
@@ -297,11 +297,11 @@ public class TestStorages {
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
- if (storeType == StoreType.RCFILE
- || storeType == StoreType.CSV
- || storeType == StoreType.PARQUET
- || storeType == StoreType.SEQUENCEFILE
- || storeType == StoreType.AVRO) {
+ if (storeType.equalsIgnoreCase("RCFILE")
+ || storeType.equalsIgnoreCase("CSV")
+ || storeType.equalsIgnoreCase("PARQUET")
+ || storeType.equalsIgnoreCase("SEQUENCEFILE")
+ || storeType.equalsIgnoreCase("AVRO")) {
assertTrue(tuple.get(0) == null);
}
assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
@@ -315,7 +315,7 @@ public class TestStorages {
@Test
public void testVariousTypes() throws IOException {
- boolean handleProtobuf = storeType != StoreType.JSON;
+ boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -336,7 +336,7 @@ public class TestStorages {
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
- if (storeType == StoreType.AVRO) {
+ if (storeType.equalsIgnoreCase("AVRO")) {
String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString();
meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
}
@@ -387,7 +387,7 @@ public class TestStorages {
@Test
public void testNullHandlingTypes() throws IOException {
- boolean handleProtobuf = storeType != StoreType.JSON;
+ boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -413,7 +413,7 @@ public class TestStorages {
meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N");
meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName());
meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\");
- if (storeType == StoreType.AVRO) {
+ if (storeType.equalsIgnoreCase("AVRO")) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA);
}
@@ -485,7 +485,7 @@ public class TestStorages {
@Test
public void testRCFileTextSerializeDeserialize() throws IOException {
- if(storeType != StoreType.RCFILE) return;
+ if(!storeType.equalsIgnoreCase("RCFILE")) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -555,7 +555,7 @@ public class TestStorages {
@Test
public void testRCFileBinarySerializeDeserialize() throws IOException {
- if(storeType != StoreType.RCFILE) return;
+ if(!storeType.equalsIgnoreCase("RCFILE")) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -625,7 +625,7 @@ public class TestStorages {
@Test
public void testSequenceFileTextSerializeDeserialize() throws IOException {
- if(storeType != StoreType.SEQUENCEFILE) return;
+ if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -699,7 +699,7 @@ public class TestStorages {
@Test
public void testSequenceFileBinarySerializeDeserialize() throws IOException {
- if(storeType != StoreType.SEQUENCEFILE) return;
+ if(!storeType.equalsIgnoreCase("SEQUENCEFILE")) return;
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@ -773,7 +773,7 @@ public class TestStorages {
@Test
public void testTime() throws IOException {
- if (storeType == StoreType.CSV || storeType == StoreType.RAW) {
+ if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("RAW")) {
Schema schema = new Schema();
schema.addColumn("col1", Type.DATE);
schema.addColumn("col2", Type.TIME);
@@ -902,11 +902,11 @@ public class TestStorages {
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- if (storeType == StoreType.AVRO) {
+ if (storeType.equalsIgnoreCase("AVRO")) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MAX_VALUE_AVRO_SCHEMA);
}
- if (storeType == StoreType.RAW) {
+ if (storeType.equalsIgnoreCase("RAW")) {
StorageManager.clearCache();
/* TAJO-1250 reproduce BufferOverflow of RAWFile */
int headerSize = 4 + 2 + 1; //Integer record length + Short null-flag length + 1 byte null flags
@@ -948,7 +948,7 @@ public class TestStorages {
scanner.close();
- if (storeType == StoreType.RAW){
+ if (storeType.equalsIgnoreCase("RAW")){
StorageManager.clearCache();
}
}
@@ -956,7 +956,7 @@ public class TestStorages {
@Test
public void testLessThanSchemaSize() throws IOException {
/* RAW is internal storage. It must be same with schema size */
- if (storeType == StoreType.RAW || storeType == StoreType.AVRO){
+ if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")){
return;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
index 7b83894..341cc07 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
@@ -22,7 +22,6 @@ import org.apache.avro.Schema;
import org.apache.tajo.HttpFileServer;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.FileUtil;
@@ -58,12 +57,12 @@ public class TestAvroUtil {
@Test
public void testGetSchema() throws IOException, URISyntaxException {
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
+ TableMeta meta = CatalogUtil.newTableMeta("AVRO");
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath())));
Schema schema = AvroUtil.getAvroSchema(meta, new TajoConf());
assertEquals(expected, schema);
- meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
+ meta = CatalogUtil.newTableMeta("AVRO");
meta.putOption(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath());
schema = AvroUtil.getAvroSchema(meta, new TajoConf());
assertEquals(expected, schema);
@@ -74,7 +73,7 @@ public class TestAvroUtil {
InetSocketAddress addr = server.getBindAddress();
String url = "http://127.0.0.1:" + addr.getPort() + schemaUrl.getPath();
- meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
+ meta = CatalogUtil.newTableMeta("AVRO");
meta.putOption(StorageConstants.AVRO_SCHEMA_URL, url);
schema = AvroUtil.getAvroSchema(meta, new TajoConf());
} finally {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
index 068f726..114a9cc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
@@ -55,9 +55,9 @@ public class TestBSTIndex {
private static final String TEST_PATH = "target/test-data/TestIndex";
private Path testDir;
private FileSystem fs;
- private StoreType storeType;
+ private String storeType;
- public TestBSTIndex(StoreType type) {
+ public TestBSTIndex(String type) {
this.storeType = type;
conf = new TajoConf();
conf.setVar(TajoConf.ConfVars.ROOT_DIR, TEST_PATH);
@@ -73,9 +73,9 @@ public class TestBSTIndex {
@Parameterized.Parameters
public static Collection<Object[]> generateParameters() {
return Arrays.asList(new Object[][]{
- {StoreType.CSV},
- {StoreType.RAW},
- {StoreType.TEXTFILE}
+ {"CSV"},
+ {"RAW"},
+ {"TEXT"}
});
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
index d7c9f49..cb94353 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
@@ -73,7 +73,7 @@ public class TestSingleCSVFileBSTIndex {
@Test
public void testFindValueInSingleCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ meta = CatalogUtil.newTableMeta("CSV");
Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv");
fs.mkdirs(tablePath.getParent());
@@ -162,7 +162,7 @@ public class TestSingleCSVFileBSTIndex {
@Test
public void testFindNextKeyValueInSingleCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ meta = CatalogUtil.newTableMeta("CSV");
Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV",
"table1.csv");
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
index 70282d9..349da07 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
@@ -68,7 +68,7 @@ public class TestJsonSerDe {
public void testVarioutType() throws IOException {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON);
+ TableMeta meta = CatalogUtil.newTableMeta("JSON");
Path tablePath = new Path(getResourcePath("dataset", "TestJsonSerDe"), "testVariousType.json");
FileSystem fs = FileSystem.getLocal(conf);
FileStatus status = fs.getFileStatus(tablePath);
[06/14] tajo git commit: TAJO-1452: Improve function listing order
(Contributed Dongjoon Hyun, Committed by hyunsik)
Posted by ji...@apache.org.
TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun, Committed by hyunsik)
Closes #470
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/1baf8dce
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/1baf8dce
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/1baf8dce
Branch: refs/heads/index_support
Commit: 1baf8dce6521fcab041c2e94b1ba5eb016611bb2
Parents: e6e1860
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri May 8 00:52:09 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri May 8 00:52:09 2015 -0700
----------------------------------------------------------------------
CHANGES | 3 ++
.../apache/tajo/function/FunctionSignature.java | 10 ++++-
.../org/apache/tajo/function/FunctionUtil.java | 39 ++++++++++++++++++++
.../cli/tsql/commands/DescFunctionCommand.java | 18 ++-------
.../main/java/org/apache/tajo/util/JSPUtil.java | 14 -------
.../main/resources/webapps/admin/functions.jsp | 3 +-
6 files changed, 55 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1baf8dce/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 15fc494..9788307 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,6 +24,9 @@ Release 0.11.0 - unreleased
IMPROVEMENT
+ TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun,
+ Committed by hyunsik)
+
TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates
an important kind of information.
(Contributed by Jongyoung Park, Committed by jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/1baf8dce/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java
index fc3a056..89ee017 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java
@@ -122,14 +122,20 @@ public class FunctionSignature implements Comparable<FunctionSignature>, ProtoOb
return cmpVal;
}
- cmpVal = returnType.getType().compareTo(o.returnType.getType());
+ cmpVal = functionType.name().compareTo(o.functionType.name());
+
+ if (cmpVal != 0) {
+ return cmpVal;
+ }
+
+ cmpVal = returnType.getType().name().compareTo(o.returnType.getType().name());
if (cmpVal != 0) {
return cmpVal;
}
for (int i = 0; i < Math.min(paramTypes.length, o.paramTypes.length); i++) {
- cmpVal = paramTypes[i].getType().compareTo(o.paramTypes[i].getType());
+ cmpVal = paramTypes[i].getType().name().compareTo(o.paramTypes[i].getType().name());
if (cmpVal != 0) {
return cmpVal;
http://git-wip-us.apache.org/repos/asf/tajo/blob/1baf8dce/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java
index ef70428..713961f 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionUtil.java
@@ -19,13 +19,52 @@
package org.apache.tajo.function;
import org.apache.tajo.catalog.FunctionDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos;
import java.util.Collection;
+import java.util.Comparator;
import static org.apache.tajo.common.TajoDataTypes.DataType;
public class FunctionUtil {
+ public static class FunctionDescProtoComparator implements Comparator<CatalogProtos.FunctionDescProto>
+ {
+ @Override
+ public int compare(CatalogProtos.FunctionDescProto f1, CatalogProtos.FunctionDescProto f2) {
+ CatalogProtos.FunctionSignatureProto s1 = f1.getSignature();
+ CatalogProtos.FunctionSignatureProto s2 = f2.getSignature();
+
+ int cmpVal = s1.getName().compareTo(s2.getName());
+
+ if (cmpVal != 0) {
+ return cmpVal;
+ }
+
+ cmpVal = s1.getType().name().compareTo(s2.getType().name());
+
+ if (cmpVal != 0) {
+ return cmpVal;
+ }
+
+ cmpVal = s1.getReturnType().getType().name().compareTo(s2.getReturnType().getType().name());
+
+ if (cmpVal != 0) {
+ return cmpVal;
+ }
+
+ for (int i = 0; i < Math.min(s1.getParameterTypesCount(), s2.getParameterTypesCount()); i++) {
+ cmpVal = s1.getParameterTypes(i).getType().name().compareTo(s2.getParameterTypes(i).getType().name());
+
+ if (cmpVal != 0) {
+ return cmpVal;
+ }
+ }
+
+ return s2.getParameterTypesCount() - s1.getParameterTypesCount();
+ }
+ }
+
public static String buildFQFunctionSignature(String funcName, DataType returnType, DataType... paramTypes) {
return returnType.getType().name().toLowerCase() + " " + buildSimpleFunctionSignature(funcName, paramTypes);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/1baf8dce/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java
index 295d326..abdbb9c 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java
@@ -52,17 +52,7 @@ public class DescFunctionCommand extends TajoShellCommand {
List<CatalogProtos.FunctionDescProto> functions =
new ArrayList<CatalogProtos.FunctionDescProto>(client.getFunctions(functionName));
- Collections.sort(functions, new Comparator<CatalogProtos.FunctionDescProto>() {
- @Override
- public int compare(CatalogProtos.FunctionDescProto f1, CatalogProtos.FunctionDescProto f2) {
- int nameCompared = f1.getSignature().getName().compareTo(f2.getSignature().getName());
- if (nameCompared != 0) {
- return nameCompared;
- } else {
- return f1.getSignature().getReturnType().getType().compareTo(f2.getSignature().getReturnType().getType());
- }
- }
- });
+ Collections.sort(functions, new FunctionUtil.FunctionDescProtoComparator());
String[] headers = new String[]{"Name", "Result type", "Argument types", "Description", "Type"};
float[] columnWidthRates = new float[]{0.15f, 0.15f, 0.2f, 0.4f, 0.1f};
@@ -70,12 +60,12 @@ public class DescFunctionCommand extends TajoShellCommand {
for(CatalogProtos.FunctionDescProto eachFunction: functions) {
String name = eachFunction.getSignature().getName();
- String resultDataType = eachFunction.getSignature().getReturnType().getType().toString();
+ String resultDataType = eachFunction.getSignature().getReturnType().getType().toString().toLowerCase();
String arguments = FunctionUtil.buildParamTypeString(
eachFunction.getSignature().getParameterTypesList().toArray(
new DataType[eachFunction.getSignature().getParameterTypesCount()]));
- String functionType = eachFunction.getSignature().getType().toString();
- String description = eachFunction.getSupplement().getShortDescription();
+ String functionType = eachFunction.getSignature().getType().toString().toLowerCase();
+ String description = eachFunction.getSupplement().getShortDescription().trim();
int index = 0;
printLeft(" " + name, columnWidths[index++]);
http://git-wip-us.apache.org/repos/asf/tajo/blob/1baf8dce/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
index 875d12b..aee2ced 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
@@ -311,20 +311,6 @@ public class JSPUtil {
}
}
- public static void sortFunctionDesc(List<FunctionDesc> functions) {
- Collections.sort(functions, new java.util.Comparator<FunctionDesc>() {
- @Override
- public int compare(FunctionDesc f1, FunctionDesc f2) {
- int nameCompared = f1.getFunctionName().compareTo(f2.getFunctionName());
- if(nameCompared != 0) {
- return nameCompared;
- } else {
- return f1.getReturnType().getType().compareTo(f2.getReturnType().getType());
- }
- }
- });
- }
-
static final DecimalFormat PERCENT_FORMAT = new DecimalFormat("###.#");
public static String percentFormat(float value) {
return PERCENT_FORMAT.format(value * 100.0f);
http://git-wip-us.apache.org/repos/asf/tajo/blob/1baf8dce/tajo-core/src/main/resources/webapps/admin/functions.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/functions.jsp b/tajo-core/src/main/resources/webapps/admin/functions.jsp
index c805aaa..cf3ddc5 100644
--- a/tajo-core/src/main/resources/webapps/admin/functions.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/functions.jsp
@@ -24,13 +24,12 @@
<%@ page import="org.apache.tajo.master.*" %>
<%@ page import="org.apache.tajo.catalog.*" %>
<%@ page import="org.apache.hadoop.http.HtmlQuoting" %>
-<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%
TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
CatalogService catalog = master.getCatalog();
List<FunctionDesc> functions = new ArrayList<FunctionDesc>(catalog.getFunctions());
- JSPUtil.sortFunctionDesc(functions);
+ Collections.sort(functions);
%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
[02/14] tajo git commit: initial work for TAJO-1591.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index ef6f1ce..1bbaa66 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -24,7 +24,6 @@ import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
@@ -108,7 +107,7 @@ public class TestFullOuterMergeJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -147,7 +146,7 @@ public class TestFullOuterMergeJoinExec {
dep4Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV");
Path dep4Path = new Path(testDir, "dep4.csv");
Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep4Meta, dep4Schema, dep4Path);
@@ -179,7 +178,7 @@ public class TestFullOuterMergeJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(job3Meta, job3Schema, job3Path);
@@ -219,7 +218,7 @@ public class TestFullOuterMergeJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(emp3Meta, emp3Schema, emp3Path);
@@ -272,7 +271,7 @@ public class TestFullOuterMergeJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV");
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(phone3Meta, phone3Schema, phone3Path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index 64efa28..30b9b19 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -82,7 +82,7 @@ public class TestHashAntiJoinExec {
employeeSchema.addColumn("memid", Type.INT4);
employeeSchema.addColumn("deptname", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -108,7 +108,7 @@ public class TestHashAntiJoinExec {
peopleSchema.addColumn("fk_memid", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV");
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(peopleMeta, peopleSchema, peoplePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index 578b586..cd42d80 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -84,7 +84,7 @@ public class TestHashJoinExec {
employeeSchema.addColumn("memid", Type.INT4);
employeeSchema.addColumn("deptname", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -107,7 +107,7 @@ public class TestHashJoinExec {
peopleSchema.addColumn("fk_memid", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV");
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(peopleMeta, peopleSchema, peoplePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index 42a018b..95debd4 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -83,7 +83,7 @@ public class TestHashSemiJoinExec {
employeeSchema.addColumn("memid", Type.INT4);
employeeSchema.addColumn("deptname", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -109,7 +109,7 @@ public class TestHashSemiJoinExec {
peopleSchema.addColumn("fk_memid", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV");
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(peopleMeta, peopleSchema, peoplePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index 5747e58..e20686b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -104,7 +104,7 @@ public class TestLeftOuterHashJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -134,7 +134,7 @@ public class TestLeftOuterHashJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(job3Meta, job3Schema, job3Path);
@@ -174,7 +174,7 @@ public class TestLeftOuterHashJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(emp3Meta, emp3Schema, emp3Path);
@@ -227,7 +227,7 @@ public class TestLeftOuterHashJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV");
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(phone3Meta, phone3Schema, phone3Path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
index ab4b881..10fd3d4 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
@@ -100,7 +100,7 @@ public class TestLeftOuterNLJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -130,7 +130,7 @@ public class TestLeftOuterNLJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(job3Meta, job3Schema, job3Path);
@@ -170,7 +170,7 @@ public class TestLeftOuterNLJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(emp3Meta, emp3Schema, emp3Path);
@@ -223,7 +223,7 @@ public class TestLeftOuterNLJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV");
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(phone3Meta, phone3Schema, phone3Path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index 1cc4e63..6e7b5dc 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -84,7 +84,7 @@ public class TestMergeJoinExec {
employeeSchema.addColumn("memid", Type.INT4);
employeeSchema.addColumn("deptname", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, employeeSchema, employeePath);
@@ -113,7 +113,7 @@ public class TestMergeJoinExec {
peopleSchema.addColumn("fk_memid", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV");
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(peopleMeta, peopleSchema, peoplePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
index 240e7fd..96c28ee 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
@@ -82,7 +82,7 @@ public class TestNLJoinExec {
schema.addColumn("memid", Type.INT4);
schema.addColumn("deptname", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, schema, employeePath);
@@ -106,7 +106,7 @@ public class TestNLJoinExec {
peopleSchema.addColumn("fk_memid", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV");
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(peopleMeta, peopleSchema, peoplePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index e2216b9..3e0b231 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -118,7 +118,7 @@ public class TestPhysicalPlanner {
scoreSchema.addColumn("score", Type.INT4);
scoreSchema.addColumn("nullable", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
@@ -139,7 +139,7 @@ public class TestPhysicalPlanner {
catalog.createTable(employee);
Path scorePath = new Path(testDir, "score");
- TableMeta scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV, new KeyValueSet());
+ TableMeta scoreMeta = CatalogUtil.newTableMeta("CSV", new KeyValueSet());
appender = sm.getAppender(scoreMeta, scoreSchema, scorePath);
appender.init();
score = new TableDesc(
@@ -180,7 +180,7 @@ public class TestPhysicalPlanner {
CommonTestingUtil.cleanupTestDir(scoreLargePath.toString());
Schema scoreSchmea = score.getSchema();
- TableMeta scoreLargeMeta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet());
+ TableMeta scoreLargeMeta = CatalogUtil.newTableMeta("RAW", new KeyValueSet());
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath);
appender.enableStats();
@@ -435,7 +435,7 @@ public class TestPhysicalPlanner {
LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta outputMeta = CatalogUtil.newTableMeta("CSV");
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -504,7 +504,7 @@ public class TestPhysicalPlanner {
long totalNum = 0;
for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) {
Scanner scanner = ((FileStorageManager)StorageManager.getFileStorageManager(conf)).getFileScanner(
- CatalogUtil.newTableMeta(StoreType.CSV),
+ CatalogUtil.newTableMeta("CSV"),
rootNode.getOutSchema(),
status.getPath());
@@ -532,7 +532,7 @@ public class TestPhysicalPlanner {
LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.RCFILE);
+ TableMeta outputMeta = CatalogUtil.newTableMeta("RCFILE");
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -742,7 +742,7 @@ public class TestPhysicalPlanner {
long expectedFileNum = (long) Math.ceil(fileVolumSum / (float)StorageUnit.MB);
assertEquals(expectedFileNum, fileStatuses.length);
}
- TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta outputMeta = CatalogUtil.newTableMeta("CSV");
Scanner scanner = new MergeScanner(conf, rootNode.getOutSchema(), outputMeta, TUtil.newList(fragments));
scanner.init();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index 161b0d8..d56c3b0 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -85,7 +85,7 @@ public class TestProgressExternalSortExec {
schema.addColumn("empid", TajoDataTypes.Type.INT4);
schema.addColumn("deptname", TajoDataTypes.Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.RAW);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("RAW");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, schema, employeePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
index 06ab6c8..097d75b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
@@ -99,7 +99,7 @@ public class TestRightOuterHashJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -129,7 +129,7 @@ public class TestRightOuterHashJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(job3Meta, job3Schema, job3Path);
@@ -169,7 +169,7 @@ public class TestRightOuterHashJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(emp3Meta, emp3Schema, emp3Path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index 30fd999..181f70e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -106,7 +106,7 @@ public class TestRightOuterMergeJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -145,7 +145,7 @@ public class TestRightOuterMergeJoinExec {
dep4Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV");
Path dep4Path = new Path(testDir, "dep4.csv");
Appender appender4 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep4Meta, dep4Schema, dep4Path);
@@ -177,7 +177,7 @@ public class TestRightOuterMergeJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(job3Meta, job3Schema, job3Path);
@@ -217,7 +217,7 @@ public class TestRightOuterMergeJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(emp3Meta, emp3Schema, emp3Path);
@@ -270,7 +270,7 @@ public class TestRightOuterMergeJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV");
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(phone3Meta, phone3Schema, phone3Path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index f2c2b1a..32746f8 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -77,7 +77,7 @@ public class TestSortExec {
schema.addColumn("empid", Type.INT4);
schema.addColumn("deptname", Type.TEXT);
- employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ employeeMeta = CatalogUtil.newTableMeta("CSV");
tablePath = StorageUtil.concatPath(workDir, "employee", "table1");
sm.getFileSystem().mkdirs(tablePath.getParent());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index f79f703..38d718e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -220,7 +220,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
assertNotNull(desc);
- assertEquals(CatalogProtos.StoreType.RCFILE, desc.getMeta().getStoreType());
+ assertTrue("RCFILE".equalsIgnoreCase(desc.getMeta().getStoreType()));
}
@Test
@@ -234,7 +234,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
assertNotNull(desc);
- assertEquals(CatalogProtos.StoreType.TEXTFILE, desc.getMeta().getStoreType());
+ assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getStoreType()));
}
@Test
@@ -248,7 +248,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1)));
assertNotNull(desc);
- assertEquals(CatalogProtos.StoreType.CSV, desc.getMeta().getStoreType());
+ assertTrue("CSV".equalsIgnoreCase(desc.getMeta().getStoreType()));
KeyValueSet options = desc.getMeta().getOptions();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
index 7ca766f..2bc16c6 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
@@ -205,7 +205,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
assertTableExists("external_hbase_mapped_table");
- HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE))
+ HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE"))
.getConnection(testingCluster.getHBaseUtil().getConf());
HTableInterface htable = hconn.getTable("external_hbase_table");
@@ -244,7 +244,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
assertTableExists("external_hbase_mapped_table");
- HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE))
+ HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE"))
.getConnection(testingCluster.getHBaseUtil().getConf());
HTableInterface htable = hconn.getTable("external_hbase_table");
@@ -297,7 +297,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
assertTableExists("external_hbase_mapped_table");
- HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE))
+ HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE"))
.getConnection(testingCluster.getHBaseUtil().getConf());
HTableInterface htable = hconn.getTable("external_hbase_table");
@@ -334,7 +334,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
assertTableExists("external_hbase_mapped_table");
- HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE))
+ HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE"))
.getConnection(testingCluster.getHBaseUtil().getConf());
HTableInterface htable = hconn.getTable("external_hbase_table");
@@ -469,7 +469,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
EvalNode evalNodeEq = new BinaryEval(EvalType.EQUAL, new FieldEval(tableDesc.getLogicalSchema().getColumn("rk")),
new ConstEval(new TextDatum("021")));
scanNode.setQual(evalNodeEq);
- StorageManager storageManager = StorageManager.getStorageManager(conf, StoreType.HBASE);
+ StorageManager storageManager = StorageManager.getStorageManager(conf, "HBASE");
List<Fragment> fragments = storageManager.getSplits("hbase_mapped_table", tableDesc, scanNode);
assertEquals(1, fragments.size());
assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
index 98e5ff9..1e4543c 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
@@ -32,7 +32,6 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -72,7 +71,7 @@ public class TestResultSet {
scoreSchema = new Schema();
scoreSchema.addColumn("deptname", Type.TEXT);
scoreSchema.addColumn("score", Type.INT4);
- scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ scoreMeta = CatalogUtil.newTableMeta("CSV");
TableStats stats = new TableStats();
Path p = sm.getTablePath("score");
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
index 712243b..3d23d16 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
@@ -69,7 +69,7 @@ public class TestExecutionBlockCursor {
tpch.loadSchemas();
tpch.loadOutSchema();
for (String table : tpch.getTableNames()) {
- TableMeta m = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
+ TableMeta m = CatalogUtil.newTableMeta("CSV");
TableDesc d = CatalogUtil.newTableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir());
TableStats stats = new TableStats();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
index 45c94a3..8591f88 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
@@ -90,7 +90,7 @@ public class TestGlobalPlanner {
tpch.loadSchemas();
tpch.loadOutSchema();
for (int i = 0; i < tables.length; i++) {
- TableMeta m = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
+ TableMeta m = CatalogUtil.newTableMeta("CSV");
TableStats stats = new TableStats();
stats.setNumBytes(volumes[i]);
TableDesc d = CatalogUtil.newTableDesc(
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
index 859a1f2..362cb24 100644
--- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
+++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
@@ -66,7 +66,7 @@ public class TestRowFile {
schema.addColumn("age", Type.INT8);
schema.addColumn("description", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.ROWFILE);
+ TableMeta meta = CatalogUtil.newTableMeta("ROWFILE");
FileStorageManager sm =
(FileStorageManager)StorageManager.getFileStorageManager(conf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java
index 6144534..8b1ca14 100644
--- a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java
+++ b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java
@@ -17,24 +17,9 @@
*/
package org.apache.tajo.ws.rs.resources;
-import java.net.URI;
-import java.util.List;
-
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
-import javax.ws.rs.client.Entity;
-import javax.ws.rs.core.GenericType;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
+import com.google.gson.internal.StringMap;
import org.apache.tajo.QueryTestCaseBase;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.storage.StorageConstants;
@@ -46,7 +31,15 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import com.google.gson.internal.StringMap;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.core.GenericType;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import java.net.URI;
+import java.util.List;
import static org.junit.Assert.*;
@@ -84,7 +77,7 @@ public class TestTablesResource extends QueryTestCaseBase {
Schema tableSchema = new Schema(new Column[] {new Column("column1", TajoDataTypes.Type.TEXT)});
KeyValueSet tableOptions = new KeyValueSet();
tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- TableMeta tableMeta = new TableMeta(StoreType.CSV, tableOptions);
+ TableMeta tableMeta = new TableMeta("CSV", tableOptions);
return new TableDesc(tableName, tableSchema, tableMeta, null);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index e0b4f7e..8b1b56d 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -1325,7 +1325,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
}
private void updatePhysicalInfo(TableDesc desc) {
- if (desc.getPath() != null && desc.getMeta().getStoreType() != StoreType.SYSTEM) {
+ if (desc.getPath() != null && desc.getMeta().getStoreType() != "SYSTEM") {
try {
Path path = new Path(desc.getPath());
FileSystem fs = path.getFileSystem(new Configuration());
@@ -1666,7 +1666,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
insertNode.setTargetLocation(new Path(expr.getLocation()));
if (expr.hasStorageType()) {
- insertNode.setStorageType(CatalogUtil.getStoreType(expr.getStorageType()));
+ insertNode.setStorageType(expr.getStorageType());
}
if (expr.hasParams()) {
KeyValueSet options = new KeyValueSet();
@@ -1740,9 +1740,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return handleCreateTableLike(context, expr, createTableNode);
if (expr.hasStorageType()) { // If storage type (using clause) is specified
- createTableNode.setStorageType(CatalogUtil.getStoreType(expr.getStorageType()));
+ createTableNode.setStorageType(expr.getStorageType());
} else { // otherwise, default type
- createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
+ createTableNode.setStorageType("CSV");
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
index 878553f..49dce87 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
@@ -272,7 +272,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
private int[] projectionCols;
private final CSVLineSerDe lineSerDe = new CSVLineSerDe();
- private final TableMeta pipeMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.TEXTFILE);
+ private final TableMeta pipeMeta = CatalogUtil.newTableMeta("TEXT");
private final Tuple EMPTY_INPUT = new VTuple(0);
private final Schema EMPTY_SCHEMA = new Schema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java
index 556ae7a..19a1820 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java
@@ -25,26 +25,24 @@ import org.apache.tajo.plan.PlanString;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
-import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-
/**
* <code>PersistentStoreNode</code> an expression for a persistent data store step.
* This includes some basic information for materializing data.
*/
public abstract class PersistentStoreNode extends UnaryNode implements Cloneable {
- @Expose protected StoreType storageType = StoreType.CSV;
+ @Expose protected String storageType = "CSV";
@Expose protected KeyValueSet options = new KeyValueSet();
protected PersistentStoreNode(int pid, NodeType nodeType) {
super(pid, nodeType);
}
- public void setStorageType(StoreType storageType) {
+ public void setStorageType(String storageType) {
this.storageType = storageType;
}
- public StoreType getStorageType() {
+ public String getStorageType() {
return this.storageType;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
index 8215f51..537abdc 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
@@ -103,7 +103,7 @@ public class ShuffleFileWriteNode extends PersistentStoreNode implements Cloneab
public String toString() {
StringBuilder sb = new StringBuilder("Shuffle Write (type=" + shuffleType.name().toLowerCase());
if (storageType != null) {
- sb.append(", storage="+ storageType.name());
+ sb.append(", storage="+ storageType);
}
sb.append(", part number=").append(numOutputs);
if (shuffleKeys != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
index 62f7db4..3a40f83 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
@@ -111,7 +111,7 @@ public class StoreTableNode extends PersistentStoreNode implements Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("Store Table (table=").append(tableName);
if (storageType != null) {
- sb.append(", storage="+ storageType.name());
+ sb.append(", storage="+ storageType);
}
sb.append(")");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 763f938..f018546 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -179,7 +179,7 @@ public class PlannerUtil {
for (LogicalNode node: scanNodes) {
scanNode = (ScanNode) node;
- isVirtualTable &= (scanNode.getTableDesc().getMeta().getStoreType() == StoreType.SYSTEM);
+ isVirtualTable &= (scanNode.getTableDesc().getMeta().getStoreType().equalsIgnoreCase("SYSTEM"));
}
return !checkIfDDLPlan(rootNode) && hasScanNode && isVirtualTable;
@@ -855,9 +855,9 @@ public class PlannerUtil {
}
public static void applySessionToTableProperties(OverridableConf sessionVars,
- StoreType storeType,
+ String storeType,
KeyValueSet tableProperties) {
- if (storeType == CSV || storeType == TEXTFILE) {
+ if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("TEXT")) {
if (sessionVars.containsKey(SessionVars.NULL_CHAR)) {
tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR));
}
@@ -890,15 +890,7 @@ public class PlannerUtil {
}
}
- public static boolean isFileStorageType(StoreType storageType) {
- if (storageType== StoreType.HBASE) {
- return false;
- } else {
- return true;
- }
- }
-
- public static StoreType getStoreType(LogicalPlan plan) {
+ public static String getStoreType(LogicalPlan plan) {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
NodeType nodeType = rootNode.getChild().getType();
if (nodeType == NodeType.CREATE_TABLE) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
index 275ab3a..99b87e9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
@@ -181,8 +181,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
private boolean assertSupportedStoreType(VerificationState state, String name) {
Preconditions.checkNotNull(name);
- CatalogProtos.StoreType storeType = CatalogUtil.getStoreType(name);
- if (storeType == null || storeType == CatalogProtos.StoreType.RAW) {
+ if (name.equalsIgnoreCase("RAW")) {
state.addVerification(String.format("Store format %s is not supported.", name));
return false;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/main/proto/Plan.proto
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto
index de949d5..e8172ba 100644
--- a/tajo-plan/src/main/proto/Plan.proto
+++ b/tajo-plan/src/main/proto/Plan.proto
@@ -217,7 +217,7 @@ message PartitionTableScanSpec {
message PersistentStoreNode {
required int32 childSeq = 1;
- required StoreType storageType = 2;
+ required string storageType = 2;
required KeyValueSetProto tableProperties = 3;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java
index bff7efb..694eeeb 100644
--- a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java
+++ b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java
@@ -44,13 +44,13 @@ public class TestLogicalNode {
GroupbyNode groupbyNode = new GroupbyNode(0);
groupbyNode.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)});
ScanNode scanNode = new ScanNode(0);
- scanNode.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));
+ scanNode.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("CSV"), new Path("in")));
GroupbyNode groupbyNode2 = new GroupbyNode(0);
groupbyNode2.setGroupingColumns(new Column[]{schema.getColumn(1), schema.getColumn(2)});
JoinNode joinNode = new JoinNode(0);
ScanNode scanNode2 = new ScanNode(0);
- scanNode2.init(CatalogUtil.newTableDesc("in2", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in2")));
+ scanNode2.init(CatalogUtil.newTableDesc("in2", schema, CatalogUtil.newTableMeta("CSV"), new Path("in2")));
groupbyNode.setChild(scanNode);
groupbyNode2.setChild(joinNode);
@@ -61,7 +61,7 @@ public class TestLogicalNode {
assertFalse(groupbyNode.deepEquals(groupbyNode2));
ScanNode scanNode3 = new ScanNode(0);
- scanNode3.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));
+ scanNode3.init(CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta("CSV"), new Path("in")));
groupbyNode2.setChild(scanNode3);
assertTrue(groupbyNode.equals(groupbyNode2));
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
index ce963c8..fb497fb 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -28,7 +28,6 @@ import org.apache.tajo.*;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.plan.LogicalPlan;
@@ -79,7 +78,7 @@ public abstract class StorageManager {
};
protected TajoConf conf;
- protected StoreType storeType;
+ protected String storeType;
/**
* Cache of StorageManager.
@@ -106,7 +105,7 @@ public abstract class StorageManager {
private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
new ConcurrentHashMap<Class<?>, Constructor<?>>();
- public StorageManager(StoreType storeType) {
+ public StorageManager(String storeType) {
this.storeType = storeType;
}
@@ -222,7 +221,7 @@ public abstract class StorageManager {
* Returns the current storage type.
* @return
*/
- public StoreType getStoreType() {
+ public String getStoreType() {
return storeType;
}
@@ -271,7 +270,7 @@ public abstract class StorageManager {
* @throws java.io.IOException
*/
public static StorageManager getFileStorageManager(TajoConf tajoConf) throws IOException {
- return getStorageManager(tajoConf, StoreType.CSV);
+ return getStorageManager(tajoConf, "CSV");
}
/**
@@ -283,22 +282,6 @@ public abstract class StorageManager {
* @throws java.io.IOException
*/
public static StorageManager getStorageManager(TajoConf tajoConf, String storeType) throws IOException {
- if ("HBASE".equalsIgnoreCase(storeType)) {
- return getStorageManager(tajoConf, StoreType.HBASE);
- } else {
- return getStorageManager(tajoConf, StoreType.CSV);
- }
- }
-
- /**
- * Returns the proper StorageManager instance according to the storeType.
- *
- * @param tajoConf Tajo system property.
- * @param storeType Storage type
- * @return
- * @throws java.io.IOException
- */
- public static StorageManager getStorageManager(TajoConf tajoConf, StoreType storeType) throws IOException {
FileSystem fileSystem = TajoConf.getWarehouseDir(tajoConf).getFileSystem(tajoConf);
if (fileSystem != null) {
return getStorageManager(tajoConf, storeType, fileSystem.getUri().toString());
@@ -317,15 +300,13 @@ public abstract class StorageManager {
* @throws java.io.IOException
*/
private static synchronized StorageManager getStorageManager (
- TajoConf tajoConf, StoreType storeType, String managerKey) throws IOException {
+ TajoConf tajoConf, String storeType, String managerKey) throws IOException {
String typeName;
- switch (storeType) {
- case HBASE:
- typeName = "hbase";
- break;
- default:
- typeName = "hdfs";
+ if (storeType.equalsIgnoreCase("HBASE")) {
+ typeName = "hbase";
+ } else {
+ typeName = "hdfs";
}
synchronized (storageManagers) {
@@ -344,7 +325,7 @@ public abstract class StorageManager {
Constructor<? extends StorageManager> constructor =
(Constructor<? extends StorageManager>) CONSTRUCTOR_CACHE.get(storageManagerClass);
if (constructor == null) {
- constructor = storageManagerClass.getDeclaredConstructor(new Class<?>[]{StoreType.class});
+ constructor = storageManagerClass.getDeclaredConstructor(new Class<?>[]{String.class});
constructor.setAccessible(true);
CONSTRUCTOR_CACHE.put(storageManagerClass, constructor);
}
@@ -449,7 +430,7 @@ public abstract class StorageManager {
Class<? extends Appender> appenderClass;
- String handlerName = CatalogUtil.getStoreTypeString(meta.getStoreType()).toLowerCase();
+ String handlerName = meta.getStoreType().toLowerCase();
appenderClass = APPENDER_HANDLER_CACHE.get(handlerName);
if (appenderClass == null) {
appenderClass = conf.getClass(
@@ -532,8 +513,8 @@ public abstract class StorageManager {
* @return The Scanner class
* @throws java.io.IOException
*/
- public Class<? extends Scanner> getScannerClass(CatalogProtos.StoreType storeType) throws IOException {
- String handlerName = CatalogUtil.getStoreTypeString(storeType).toLowerCase();
+ public Class<? extends Scanner> getScannerClass(String storeType) throws IOException {
+ String handlerName = storeType.toLowerCase();
Class<? extends Scanner> scannerClass = SCANNER_HANDLER_CACHE.get(handlerName);
if (scannerClass == null) {
scannerClass = conf.getClass(
@@ -542,7 +523,7 @@ public abstract class StorageManager {
}
if (scannerClass == null) {
- throw new IOException("Unknown Storage Type: " + storeType.name());
+ throw new IOException("Unknown Storage Type: " + storeType);
}
return scannerClass;
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
index e95aeec..32f1e43 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
@@ -23,7 +23,6 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.logical.LogicalNode;
@@ -55,7 +54,7 @@ public class AddSortForInsertRewriter implements LogicalPlanRewriteRule {
@Override
public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
- StoreType storeType = PlannerUtil.getStoreType(plan);
+ String storeType = PlannerUtil.getStoreType(plan);
return storeType != null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
index 45a1bff..7f3cb04 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.storage.StorageManager;
@@ -48,7 +47,7 @@ public class HBasePutAppender extends AbstractHBaseAppender {
super.init();
Configuration hbaseConf = HBaseStorageManager.getHBaseConfiguration(conf, meta);
- HConnection hconn = ((HBaseStorageManager) StorageManager.getStorageManager((TajoConf)conf, StoreType.HBASE))
+ HConnection hconn = ((HBaseStorageManager) StorageManager.getStorageManager((TajoConf)conf, "HBASE"))
.getConnection(hbaseConf);
htable = hconn.getTable(columnMapping.getHbaseTableName());
htable.setAutoFlushTo(false);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
index ab56252..4303b59 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
@@ -179,7 +178,7 @@ public class HBaseScanner implements Scanner {
}
if (htable == null) {
- HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, StoreType.HBASE))
+ HConnection hconn = ((HBaseStorageManager)StorageManager.getStorageManager(conf, "HBASE"))
.getConnection(hbaseConf);
htable = hconn.getTable(fragment.getHbaseTableName());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
index a9e5bde..a7c579a 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.tajo.*;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
@@ -70,7 +69,7 @@ public class HBaseStorageManager extends StorageManager {
private Map<HConnectionKey, HConnection> connMap = new HashMap<HConnectionKey, HConnection>();
- public HBaseStorageManager (StoreType storeType) {
+ public HBaseStorageManager (String storeType) {
super(storeType);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java
index 68939d6..dd52324 100644
--- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java
+++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java
@@ -20,7 +20,6 @@ package org.apache.tajo.storage.hbase;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.util.KeyValueSet;
import org.junit.Test;
@@ -42,7 +41,7 @@ public class TestColumnMapping {
schema.addColumn("c3", Type.TEXT);
schema.addColumn("c4", Type.TEXT);
- TableMeta tableMeta = new TableMeta(StoreType.HBASE, keyValueSet);
+ TableMeta tableMeta = new TableMeta("HBASE", keyValueSet);
ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java
index 1fc4065..aa7aa28 100644
--- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java
@@ -48,7 +48,7 @@ public class TestHBaseStorageManager {
scanNode.setQual(evalNodeA);
HBaseStorageManager storageManager =
- (HBaseStorageManager) StorageManager.getStorageManager(new TajoConf(), StoreType.HBASE);
+ (HBaseStorageManager) StorageManager.getStorageManager(new TajoConf(), "HBASE");
List<Set<EvalNode>> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
assertNotNull(indexEvals);
assertEquals(1, indexEvals.size());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
index 8d425b4..fd4ab84 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
@@ -28,10 +28,9 @@ import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoConstants;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.plan.logical.LogicalNode;
@@ -86,7 +85,7 @@ public class FileStorageManager extends StorageManager {
protected boolean blocksMetadataEnabled;
private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0));
- public FileStorageManager(StoreType storeType) {
+ public FileStorageManager(String storeType) {
super(storeType);
}
@@ -855,7 +854,7 @@ public class FileStorageManager extends StorageManager {
public StorageProperty getStorageProperty() {
StorageProperty storageProperty = new StorageProperty();
storageProperty.setSortedInsert(false);
- if (storeType == StoreType.RAW) {
+ if (storeType.equalsIgnoreCase("RAW")) {
storageProperty.setSupportsInsertInto(false);
} else {
storageProperty.setSupportsInsertInto(true);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
index 3c78d6b..68a2cf2 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
@@ -54,11 +54,11 @@ public class TestCompressionStorages {
private TajoConf conf;
private static String TEST_PATH = "target/test-data/TestCompressionStorages";
- private StoreType storeType;
+ private String storeType;
private Path testDir;
private FileSystem fs;
- public TestCompressionStorages(StoreType type) throws IOException {
+ public TestCompressionStorages(String type) throws IOException {
this.storeType = type;
conf = new TajoConf();
@@ -69,10 +69,10 @@ public class TestCompressionStorages {
@Parameterized.Parameters
public static Collection<Object[]> generateParameters() {
return Arrays.asList(new Object[][]{
- {StoreType.CSV},
- {StoreType.RCFILE},
- {StoreType.SEQUENCEFILE},
- {StoreType.TEXTFILE}
+ {"CSV"},
+ {"RCFILE"},
+ {"SEQUENCEFILE"},
+ {"TEXT"}
});
}
@@ -83,11 +83,11 @@ public class TestCompressionStorages {
@Test
public void testGzipCodecCompressionData() throws IOException {
- if (storeType == StoreType.RCFILE) {
+ if (storeType.equalsIgnoreCase("RCFILE")) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(storeType, GzipCodec.class);
}
- } else if (storeType == StoreType.SEQUENCEFILE) {
+ } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) {
if( ZlibFactory.isNativeZlibLoaded(conf)) {
storageCompressionTest(storeType, GzipCodec.class);
}
@@ -109,7 +109,7 @@ public class TestCompressionStorages {
storageCompressionTest(storeType, Lz4Codec.class);
}
- private void storageCompressionTest(StoreType storeType, Class<? extends CompressionCodec> codec) throws IOException {
+ private void storageCompressionTest(String storeType, Class<? extends CompressionCodec> codec) throws IOException {
Schema schema = new Schema();
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.FLOAT4);
@@ -157,7 +157,7 @@ public class TestCompressionStorages {
Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
- if (StoreType.CSV == storeType) {
+ if (storeType.equalsIgnoreCase("CSV")) {
if (SplittableCompressionCodec.class.isAssignableFrom(codec)) {
assertTrue(scanner.isSplittable());
} else {
@@ -166,7 +166,7 @@ public class TestCompressionStorages {
}
scanner.init();
- if (storeType == StoreType.SEQUENCEFILE) {
+ if (storeType.equalsIgnoreCase("SEQUENCEFILE")) {
assertTrue(scanner instanceof SequenceFileScanner);
Writable key = ((SequenceFileScanner) scanner).getKey();
assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
index 2db916a..6e15c51 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
@@ -101,7 +101,7 @@ public class TestDelimitedTextFile {
public void testIgnoreAllErrors() throws IOException {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON);
+ TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "-1");
FileFragment fragment = getFileFragment("testErrorTolerance1.json");
Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
@@ -123,7 +123,7 @@ public class TestDelimitedTextFile {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON);
+ TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1");
FileFragment fragment = getFileFragment("testErrorTolerance1.json");
Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
@@ -145,7 +145,7 @@ public class TestDelimitedTextFile {
@Test
public void testNoErrorTolerance() throws IOException {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON);
+ TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "0");
FileFragment fragment = getFileFragment("testErrorTolerance2.json");
Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
@@ -164,7 +164,7 @@ public class TestDelimitedTextFile {
@Test
public void testIgnoreTruncatedValueErrorTolerance() throws IOException {
TajoConf conf = new TajoConf();
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON);
+ TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1");
FileFragment fragment = getFileFragment("testErrorTolerance3.json");
Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
index 4b39cdd..7d5eee1 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
@@ -68,7 +68,7 @@ public class TestFileStorageManager {
schema.addColumn("age",Type.INT4);
schema.addColumn("name",Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
Tuple[] tuples = new Tuple[4];
for(int i=0; i < tuples.length; i++) {
@@ -135,7 +135,7 @@ public class TestFileStorageManager {
schema.addColumn("id", Type.INT4);
schema.addColumn("age",Type.INT4);
schema.addColumn("name",Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
List<Fragment> splits = Lists.newArrayList();
// Get FileFragments in partition batch
@@ -189,7 +189,7 @@ public class TestFileStorageManager {
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT4);
schema.addColumn("name", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
List<Fragment> splits = Lists.newArrayList();
splits.addAll(sm.getSplits("data", meta, schema, tablePath));
@@ -221,11 +221,11 @@ public class TestFileStorageManager {
try {
/* Local FileSystem */
- FileStorageManager sm = (FileStorageManager)StorageManager.getStorageManager(conf, StoreType.CSV);
+ FileStorageManager sm = (FileStorageManager)StorageManager.getStorageManager(conf, "CSV");
assertEquals(fs.getUri(), sm.getFileSystem().getUri());
/* Distributed FileSystem */
- sm = (FileStorageManager)StorageManager.getStorageManager(tajoConf, StoreType.CSV);
+ sm = (FileStorageManager)StorageManager.getStorageManager(tajoConf, "CSV");
assertNotEquals(fs.getUri(), sm.getFileSystem().getUri());
assertEquals(cluster.getFileSystem().getUri(), sm.getFileSystem().getUri());
} finally {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
index ff7fe13..b4a60fc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
@@ -103,7 +103,7 @@ public class TestFileSystems {
schema.addColumn("age", Type.INT4);
schema.addColumn("name", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
Tuple[] tuples = new Tuple[4];
for (int i = 0; i < tuples.length; i++) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
index f405734..1078b84 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
@@ -64,7 +64,7 @@ public class TestLineReader {
schema.addColumn("comment", Type.TEXT);
schema.addColumn("comment2", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.TEXTFILE);
+ TableMeta meta = CatalogUtil.newTableMeta("TEXT");
Path tablePath = new Path(testDir, "line.data");
FileAppender appender = (FileAppender) StorageManager.getFileStorageManager(conf).getAppender(
null, null, meta, schema, tablePath);
@@ -115,7 +115,7 @@ public class TestLineReader {
schema.addColumn("comment", Type.TEXT);
schema.addColumn("comment2", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.TEXTFILE);
+ TableMeta meta = CatalogUtil.newTableMeta("TEXT");
meta.putOption("compression.codec", DeflateCodec.class.getCanonicalName());
Path tablePath = new Path(testDir, "testLineDelimitedReaderWithCompression." + DeflateCodec.class.getSimpleName());
@@ -174,7 +174,7 @@ public class TestLineReader {
schema.addColumn("comment", Type.TEXT);
schema.addColumn("comment2", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.TEXTFILE);
+ TableMeta meta = CatalogUtil.newTableMeta("TEXT");
Path tablePath = new Path(testDir, "testLineDelimitedReader");
FileAppender appender = (FileAppender) StorageManager.getFileStorageManager(conf).getAppender(
@@ -278,7 +278,7 @@ public class TestLineReader {
schema.addColumn("comment", Type.TEXT);
schema.addColumn("comment2", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.TEXTFILE);
+ TableMeta meta = CatalogUtil.newTableMeta("TEXT");
Path tablePath = new Path(testDir, "testSeekableByteBufLineReader.data");
FileAppender appender = (FileAppender) StorageManager.getFileStorageManager(conf).getAppender(
null, null, meta, schema, tablePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index 561e2ef..e2cfe8f 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@ -67,22 +67,22 @@ public class TestMergeScanner {
"}\n";
private Path testDir;
- private StoreType storeType;
+ private String storeType;
private FileSystem fs;
- public TestMergeScanner(StoreType storeType) {
+ public TestMergeScanner(String storeType) {
this.storeType = storeType;
}
@Parameters
public static Collection<Object[]> generateParameters() {
return Arrays.asList(new Object[][] {
- {StoreType.CSV},
- {StoreType.RAW},
- {StoreType.RCFILE},
- {StoreType.PARQUET},
- {StoreType.SEQUENCEFILE},
- {StoreType.AVRO},
+ {"CSV"},
+ {"RAW"},
+ {"RCFILE"},
+ {"PARQUET"},
+ {"SEQUENCEFILE"},
+ {"AVRO"},
// RowFile requires Byte-buffer read support, so we omitted RowFile.
//{StoreType.ROWFILE},
});
@@ -109,7 +109,7 @@ public class TestMergeScanner {
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
- if (storeType == StoreType.AVRO) {
+ if (storeType.equalsIgnoreCase("AVRO")) {
meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
TEST_MULTIPLE_FILES_AVRO_SCHEMA);
}
@@ -187,16 +187,15 @@ public class TestMergeScanner {
assertEquals(tupleNum * 2, totalCounts);
}
- private static boolean isProjectableStorage(StoreType type) {
- switch (type) {
- case RCFILE:
- case PARQUET:
- case SEQUENCEFILE:
- case CSV:
- case AVRO:
- return true;
- default:
- return false;
+ private static boolean isProjectableStorage(String type) {
+ if (type.equalsIgnoreCase("RCFILE") ||
+ type.equalsIgnoreCase("PARQUET") ||
+ type.equalsIgnoreCase("SEQUENCEFILE") ||
+ type.equalsIgnoreCase("CSV") ||
+ type.equalsIgnoreCase("AVRO")) {
+ return true;
+ } else {
+ return false;
}
}
}
[04/14] tajo git commit: TAJO-1576: Sometimes
DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates an important
kind of information.
Posted by ji...@apache.org.
TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates an important kind of information.
Closes #561
Signed-off-by: Jihoon Son <ji...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/e6e1860c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/e6e1860c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/e6e1860c
Branch: refs/heads/index_support
Commit: e6e1860c8e83f9db6c4c094fe703b21d45d86529
Parents: 2b4c161
Author: Jongyoung Park <em...@gmail.com>
Authored: Fri May 8 15:10:09 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Fri May 8 15:10:09 2015 +0900
----------------------------------------------------------------------
CHANGES | 4 ++++
.../tajo/cli/tsql/DefaultTajoCliOutputFormatter.java | 4 +---
.../tajo/cli/tsql/TestDefaultCliOutputFormatter.java | 13 +++++++++----
3 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/e6e1860c/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index d448c09..15fc494 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,6 +24,10 @@ Release 0.11.0 - unreleased
IMPROVEMENT
+ TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates
+ an important kind of information.
+ (Contributed by Jongyoung Park, Committed by jihoon)
+
TAJO-1408: Make IntermediateEntryProto more compact.
(Contributed by navis, Committed by hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/e6e1860c/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
index aaa411e..03aa4cc 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
@@ -196,10 +196,8 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
if (message == null) {
return TajoCli.ERROR_PREFIX + "No error message";
}
- String[] lines = message.split("\n");
- message = lines[0];
- int index = message.lastIndexOf(TajoCli.ERROR_PREFIX);
+ int index = message.indexOf(TajoCli.ERROR_PREFIX);
if (index < 0) {
message = TajoCli.ERROR_PREFIX + message;
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/e6e1860c/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java
index 0bb11e0..f6dcaec 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java
@@ -97,13 +97,19 @@ public class TestDefaultCliOutputFormatter {
"Caused by: java.sql.SQLException: ERROR: no such a table: table1\n" +
"\t... 6 more";
- assertEquals("ERROR: no such a table: table1", DefaultTajoCliOutputFormatter.parseErrorMessage(multiLineMessage));
+ assertEquals(multiLineMessage, DefaultTajoCliOutputFormatter.parseErrorMessage(multiLineMessage));
+
+ String noPrefixMessage = "RTFM please";
+ assertEquals("ERROR: "+noPrefixMessage, DefaultTajoCliOutputFormatter.parseErrorMessage(noPrefixMessage));
+
+ String errorMessageWithLine = "ERROR: syntax error at or near '('\n" +
+ "LINE 1:7 select (*) from tc\n" +
+ " ^";
+ assertEquals(errorMessageWithLine, DefaultTajoCliOutputFormatter.parseErrorMessage(errorMessageWithLine));
}
@Test
public void testPrintResultInsertStatement() throws Exception {
-
-
DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter();
outputFormatter.init(cliContext);
@@ -172,5 +178,4 @@ public class TestDefaultCliOutputFormatter {
assertEquals(expectedOutput, stringWriter.toString());
}
-
}
[14/14] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/6f440962
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/6f440962
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/6f440962
Branch: refs/heads/index_support
Commit: 6f4409629d4bcca6487ab17905b375648938cdb0
Parents: 410ca31 53ed1c3
Author: Jihoon Son <ji...@apache.org>
Authored: Tue May 12 12:13:43 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue May 12 12:13:43 2015 +0900
----------------------------------------------------------------------
CHANGES | 14 +
.../org/apache/tajo/catalog/CatalogUtil.java | 36 +-
.../org/apache/tajo/catalog/DDLBuilder.java | 6 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++
.../java/org/apache/tajo/catalog/Schema.java | 109 +++++-
.../org/apache/tajo/catalog/SchemaUtil.java | 33 +-
.../java/org/apache/tajo/catalog/TableDesc.java | 5 +-
.../java/org/apache/tajo/catalog/TableMeta.java | 8 +-
.../tajo/catalog/json/TableMetaAdapter.java | 7 +-
.../apache/tajo/function/FunctionSignature.java | 10 +-
.../org/apache/tajo/function/FunctionUtil.java | 39 +++
.../src/main/proto/CatalogProtos.proto | 2 +-
.../org/apache/tajo/catalog/TestTableDesc.java | 5 +-
.../org/apache/tajo/catalog/TestTableMeta.java | 12 +-
.../tajo/catalog/store/HiveCatalogStore.java | 26 +-
.../catalog/store/TestHiveCatalogStore.java | 22 +-
.../dictionary/AbstractTableDescriptor.java | 2 +-
.../tajo/catalog/store/AbstractDBStore.java | 8 +-
.../org/apache/tajo/catalog/store/MemStore.java | 2 +-
.../org/apache/tajo/catalog/TestCatalog.java | 24 +-
.../org/apache/tajo/cli/tools/TajoDump.java | 2 +-
.../cli/tsql/DefaultTajoCliOutputFormatter.java | 4 +-
.../cli/tsql/commands/DescFunctionCommand.java | 18 +-
.../cli/tsql/commands/DescTableCommand.java | 7 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/BuiltinStorages.java | 32 ++
.../java/org/apache/tajo/storage/Tuple.java | 2 +-
.../java/org/apache/tajo/util/StringUtils.java | 70 ++++
.../main/java/org/apache/tajo/util/TUtil.java | 32 --
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../java/org/apache/tajo/benchmark/TPCH.java | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +-
.../engine/planner/PhysicalPlannerImpl.java | 30 +-
.../tajo/engine/planner/global/DataChannel.java | 10 +-
.../engine/planner/global/GlobalPlanner.java | 16 +-
.../global/builder/DistinctGroupbyBuilder.java | 6 +-
.../DistinctGroupbyThirdAggregationExec.java | 2 +-
.../planner/physical/ExternalSortExec.java | 4 +-
.../planner/physical/PhysicalPlanUtil.java | 56 ++--
.../physical/RangeShuffleFileWriteExec.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 27 +-
.../tajo/engine/utils/TupleCacheScanner.java | 2 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 8 +-
.../org/apache/tajo/master/GlobalEngine.java | 2 +-
.../apache/tajo/master/exec/DDLExecutor.java | 3 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../NonForwardQueryResultSystemScanner.java | 22 +-
.../apache/tajo/master/exec/QueryExecutor.java | 6 +-
.../java/org/apache/tajo/querymaster/Query.java | 4 +-
.../tajo/querymaster/QueryMasterTask.java | 2 +-
.../apache/tajo/querymaster/Repartitioner.java | 4 +-
.../java/org/apache/tajo/querymaster/Stage.java | 4 +-
.../main/java/org/apache/tajo/util/JSPUtil.java | 14 -
.../tajo/worker/ExecutionBlockContext.java | 15 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 1 +
.../src/main/proto/TajoWorkerProtocol.proto | 2 +-
.../resources/webapps/admin/catalogview.jsp | 5 +-
.../main/resources/webapps/admin/functions.jsp | 3 +-
.../org/apache/tajo/BackendTestingUtil.java | 2 +-
.../apache/tajo/LocalTajoTestingUtility.java | 2 +-
.../org/apache/tajo/TajoTestingCluster.java | 2 +-
.../apache/tajo/cli/tools/TestDDLBuilder.java | 2 +-
.../cli/tsql/TestDefaultCliOutputFormatter.java | 13 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 4 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 2 +-
.../engine/function/TestFunctionLoader.java | 5 +-
.../engine/planner/TestLogicalOptimizer.java | 6 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 18 +-
.../tajo/engine/planner/TestPlannerUtil.java | 9 +-
.../planner/global/TestBroadcastJoinPlan.java | 2 +-
.../planner/physical/TestBNLJoinExec.java | 4 +-
.../planner/physical/TestExternalSortExec.java | 2 +-
.../physical/TestFullOuterHashJoinExec.java | 8 +-
.../physical/TestFullOuterMergeJoinExec.java | 11 +-
.../planner/physical/TestHashAntiJoinExec.java | 4 +-
.../planner/physical/TestHashJoinExec.java | 4 +-
.../planner/physical/TestHashSemiJoinExec.java | 4 +-
.../physical/TestLeftOuterHashJoinExec.java | 8 +-
.../physical/TestLeftOuterNLJoinExec.java | 8 +-
.../planner/physical/TestMergeJoinExec.java | 4 +-
.../engine/planner/physical/TestNLJoinExec.java | 4 +-
.../planner/physical/TestPhysicalPlanner.java | 14 +-
.../physical/TestProgressExternalSortExec.java | 2 +-
.../physical/TestRightOuterHashJoinExec.java | 6 +-
.../physical/TestRightOuterMergeJoinExec.java | 10 +-
.../engine/planner/physical/TestSortExec.java | 2 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 10 +-
.../tajo/engine/query/TestHBaseTable.java | 10 +-
.../tajo/engine/query/TestJoinBroadcast.java | 2 +-
.../engine/query/TestSelectNestedRecord.java | 71 ++++
.../org/apache/tajo/jdbc/TestResultSet.java | 3 +-
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +-
.../tajo/master/TestExecutionBlockCursor.java | 2 +-
.../apache/tajo/master/TestGlobalPlanner.java | 2 +-
.../org/apache/tajo/storage/TestRowFile.java | 2 +-
.../rs/resources/TestQueryResultResource.java | 6 +-
.../ws/rs/resources/TestTablesResource.java | 31 +-
.../TestSelectNestedRecord/sample1/table.json | 3 +
.../TestSelectNestedRecord/tweets/sample1.json | 4 +
.../TestSelectNestedRecord/sample1_ddl.sql | 7 +
.../TestSelectNestedRecord/sample2_ddl.sql | 19 ++
.../testNestedFieldAsGroupbyKey1.sql | 7 +
.../testNestedFieldAsJoinKey1.sql | 7 +
.../TestSelectNestedRecord/testSelect1.sql | 1 +
.../TestSelectNestedRecord/testSelect2.sql | 61 ++++
.../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++
.../testSelectWithParentheses2.sql | 2 +-
.../testNestedFieldAsGroupbyKey1.result | 6 +
.../testNestedFieldAsJoinKey1.result | 6 +
.../TestSelectNestedRecord/testSelect1.result | 5 +
.../TestSelectNestedRecord/testSelect2.result | 6 +
.../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +-
.../tajo/plan/LogicalPlanPreprocessor.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 41 ++-
.../apache/tajo/plan/expr/RowConstantEval.java | 3 +-
.../tajo/plan/expr/WindowFunctionEval.java | 3 +-
.../function/python/PythonScriptEngine.java | 14 +-
.../GreedyHeuristicJoinOrderAlgorithm.java | 8 +-
.../apache/tajo/plan/joinorder/JoinEdge.java | 4 +-
.../apache/tajo/plan/joinorder/JoinGraph.java | 6 +-
.../tajo/plan/logical/DistinctGroupbyNode.java | 4 +-
.../apache/tajo/plan/logical/EvalExprNode.java | 3 +-
.../apache/tajo/plan/logical/GroupbyNode.java | 10 +-
.../tajo/plan/logical/PersistentStoreNode.java | 8 +-
.../tajo/plan/logical/ProjectionNode.java | 3 +-
.../apache/tajo/plan/logical/RelationNode.java | 12 +
.../tajo/plan/logical/ShuffleFileWriteNode.java | 5 +-
.../tajo/plan/logical/StoreTableNode.java | 2 +-
.../tajo/plan/logical/TruncateTableNode.java | 4 +-
.../apache/tajo/plan/logical/WindowAggNode.java | 7 +-
.../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++---
.../plan/nameresolver/ResolverByLegacy.java | 8 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 4 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 18 +-
.../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +-
.../plan/verifier/PreLogicalPlanVerifier.java | 9 +-
tajo-plan/src/main/proto/Plan.proto | 2 +-
.../org/apache/tajo/plan/TestLogicalNode.java | 6 +-
.../org/apache/tajo/rpc/AsyncRpcServer.java | 6 +-
.../org/apache/tajo/rpc/BlockingRpcServer.java | 6 +-
.../org/apache/tajo/storage/MergeScanner.java | 2 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 9 +
.../org/apache/tajo/storage/StorageManager.java | 51 +--
.../storage/hbase/AbstractHBaseAppender.java | 2 -
.../storage/hbase/AddSortForInsertRewriter.java | 4 +-
.../tajo/storage/hbase/ColumnMapping.java | 2 +-
.../HBaseBinarySerializerDeserializer.java | 10 +-
.../tajo/storage/hbase/HBasePutAppender.java | 3 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 25 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 9 +-
.../tajo/storage/hbase/TestColumnMapping.java | 3 +-
.../storage/hbase/TestHBaseStorageManager.java | 2 +-
.../java/org/apache/tajo/storage/CSVFile.java | 2 +-
.../org/apache/tajo/storage/FileScanner.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 9 +-
.../apache/tajo/storage/avro/AvroScanner.java | 25 +-
.../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++--------
.../apache/tajo/storage/json/JsonLineSerDe.java | 5 +-
.../tajo/storage/json/JsonLineSerializer.java | 2 -
.../storage/parquet/TajoRecordConverter.java | 7 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 16 +-
.../tajo/storage/text/CSVLineDeserializer.java | 26 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 5 +-
.../tajo/storage/text/DelimitedTextFile.java | 11 +-
.../tajo/storage/text/TextLineDeserializer.java | 4 +-
.../apache/tajo/storage/text/TextLineSerDe.java | 3 +-
.../tajo/storage/TestCompressionStorages.java | 22 +-
.../tajo/storage/TestDelimitedTextFile.java | 8 +-
.../tajo/storage/TestFileStorageManager.java | 10 +-
.../apache/tajo/storage/TestFileSystems.java | 2 +-
.../org/apache/tajo/storage/TestLineReader.java | 8 +-
.../apache/tajo/storage/TestMergeScanner.java | 49 +--
.../org/apache/tajo/storage/TestStorages.java | 86 ++---
.../apache/tajo/storage/avro/TestAvroUtil.java | 7 +-
.../apache/tajo/storage/index/TestBSTIndex.java | 10 +-
.../index/TestSingleCSVFileBSTIndex.java | 4 +-
.../apache/tajo/storage/json/TestJsonSerDe.java | 2 +-
180 files changed, 1741 insertions(+), 905 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 67d656f,73fd2f3..42d9b36
--- a/CHANGES
+++ b/CHANGES
@@@ -114,8 -123,8 +123,10 @@@ Release 0.11.0 - unrelease
BUG FIXES
+ TAJO-1594: Catalog schema is invalid for some databases. (jihoon)
+
+ TAJO-1593: Add missing stop condition to Taskrunner. (jinho)
+
TAJO-1556: "insert into select" with reordered column list does not work.
(Contributed by Yongjin Choi, Committed by jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index f3f71ba,17f9146..e44076a
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@@ -22,9 -22,11 +22,8 @@@ import com.google.common.base.Objects
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.Expose;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
- import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.ProtoObject;
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 1c17cae,cbcec83..884b97d
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@@ -547,9 -561,9 +547,9 @@@ public class TestCatalog
String tableName = "indexed";
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
return new TableDesc(
- CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta,
+ CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), relationSchema, meta,
new Path(CommonTestingUtil.getTestDir(), "indexed").toUri());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index cbe17b7,6df26b7..31dfb13
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@@ -24,12 -24,9 +24,11 @@@ import org.apache.tajo.TajoConstants
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
import org.apache.tajo.cli.tsql.TajoCli;
import org.apache.tajo.util.FileUtil;
- import org.apache.tajo.util.TUtil;
+ import org.apache.tajo.util.StringUtils;
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index 313e6a3,506b03e..2dd7fa8
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@@ -30,13 -30,12 +30,10 @@@ import org.apache.commons.logging.LogFa
import org.apache.hadoop.fs.Path;
import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Column;
--import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
import org.apache.tajo.conf.TajoConf;
- import org.apache.tajo.datum.Datum;
--import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.physical.*;
@@@ -49,13 -48,14 +46,16 @@@ import org.apache.tajo.ipc.TajoWorkerPr
import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.*;
- import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
++import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
import org.apache.tajo.plan.util.PlannerUtil;
--import org.apache.tajo.storage.*;
++import org.apache.tajo.storage.FileStorageManager;
++import org.apache.tajo.storage.StorageConstants;
++import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.IndexUtil;
+ import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
index 9fb75db,6ae9677..1150007
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
@@@ -293,35 -289,43 +293,35 @@@ public class NonForwardQueryResultSyste
return tuples;
}
-
+
private List<Tuple> getIndexes(Schema outSchema) {
- List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes();
+ List<IndexDescProto> indexList = masterContext.getCatalog().getAllIndexes();
List<Tuple> tuples = new ArrayList<Tuple>(indexList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
-
- for (IndexProto index: indexList) {
+
+ for (IndexDescProto index: indexList) {
aTuple = new VTuple(outSchema.size());
-
+
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
-
+
if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt4(index.getDbId()));
+ aTuple.put(fieldId, DatumFactory.createInt4(index.getTableIdentifier().getDbId()));
} else if ("tid".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt4(index.getTId()));
+ aTuple.put(fieldId, DatumFactory.createInt4(index.getTableIdentifier().getTid()));
} else if ("index_name".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(index.getIndexName()));
- } else if ("column_name".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(index.getColumnName()));
- } else if ("data_type".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(index.getDataType()));
- } else if ("index_type".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(index.getIndexType()));
- } else if ("is_unique".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createBool(index.getIsUnique()));
- } else if ("is_clustered".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createBool(index.getIsClustered()));
- } else if ("is_ascending".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createBool(index.getIsAscending()));
+ } else if ("index_method".equalsIgnoreCase(column.getSimpleName())) {
+ aTuple.put(fieldId, DatumFactory.createText(index.getIndexMethod().name()));
+ } else if ("index_path".equalsIgnoreCase(column.getSimpleName())) {
+ aTuple.put(fieldId, DatumFactory.createText(index.getIndexPath()));
}
}
-
+
tuples.add(aTuple);
}
-
+
return tuples;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 0ea1e20,ce1e4ec..4d1d7a8
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@@ -36,10 -36,8 +36,9 @@@ import org.apache.tajo.algebra.WindowSp
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
- import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.plan.LogicalPlan.QueryBlock;
import org.apache.tajo.plan.algebra.BaseAlgebraVisitor;
@@@ -53,9 -51,10 +52,10 @@@ import org.apache.tajo.plan.util.Planne
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
+ import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
+import java.net.URI;
import java.util.*;
import static org.apache.tajo.algebra.CreateTable.PartitionType;
@@@ -879,11 -878,17 +890,11 @@@ public class LogicalPlanner extends Bas
} else if (block.namedExprsMgr.isEvaluated(refName)) {
column = block.namedExprsMgr.getTarget(refName).getNamedColumn();
} else {
- throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(rawSortSpecs));
- throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs));
++ throw new IllegalStateException("Unexpected State: " + StringUtils.join(rawSortSpecs));
}
- annotatedSortSpecs.add(new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst()));
- }
-
- if (annotatedSortSpecs.size() == 0) {
- return child;
- } else {
- sortNode.setSortSpecs(annotatedSortSpecs.toArray(new SortSpec[annotatedSortSpecs.size()]));
- return sortNode;
+ annotatedSortSpecs.add(new SortSpec(column, rawSortSpecs[i].isAscending(), rawSortSpecs[i].isNullFirst()));
}
+ return annotatedSortSpecs.toArray(new SortSpec[annotatedSortSpecs.size()]);
}
/*===============================================================================================
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-plan/src/main/proto/Plan.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/6f440962/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
----------------------------------------------------------------------
diff --cc tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
index f9a57fa,32f1e43..c380a23
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
@@@ -18,7 -18,7 +18,6 @@@
package org.apache.tajo.storage.hbase;
--import org.apache.tajo.OverridableConf;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
@@@ -55,8 -53,8 +53,8 @@@ public class AddSortForInsertRewriter i
}
@Override
- public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
- String storeType = PlannerUtil.getStoreType(plan);
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
- StoreType storeType = PlannerUtil.getStoreType(context.getPlan());
++ String storeType = PlannerUtil.getStoreType(context.getPlan());
return storeType != null;
}
[13/14] tajo git commit: TAJO-1591: Change StoreType represented as
Enum to String type. (hyunsik)
Posted by ji...@apache.org.
TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik)
Closes #564
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/53ed1c37
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/53ed1c37
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/53ed1c37
Branch: refs/heads/index_support
Commit: 53ed1c376f4969b5a2c54e30ddbd8113847a8768
Parents: 70df3a8 0d1bf41
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon May 11 15:29:17 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon May 11 15:29:17 2015 -0700
----------------------------------------------------------------------
CHANGES | 11 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 12 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++
.../java/org/apache/tajo/catalog/Schema.java | 109 +++++-
.../org/apache/tajo/catalog/SchemaUtil.java | 33 +-
.../tajo/catalog/store/HiveCatalogStore.java | 4 +-
.../cli/tsql/commands/DescTableCommand.java | 4 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Tuple.java | 2 +-
.../java/org/apache/tajo/util/StringUtils.java | 70 ++++
.../main/java/org/apache/tajo/util/TUtil.java | 32 --
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +-
.../engine/planner/PhysicalPlannerImpl.java | 17 +-
.../tajo/engine/planner/global/DataChannel.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 6 +-
.../global/builder/DistinctGroupbyBuilder.java | 6 +-
.../DistinctGroupbyThirdAggregationExec.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 27 +-
.../tajo/engine/utils/TupleCacheScanner.java | 2 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 8 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../NonForwardQueryResultSystemScanner.java | 20 +-
.../tajo/worker/ExecutionBlockContext.java | 15 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 1 +
.../resources/webapps/admin/catalogview.jsp | 5 +-
.../engine/function/TestFunctionLoader.java | 5 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 8 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 4 +-
.../tajo/engine/query/TestJoinBroadcast.java | 2 +-
.../engine/query/TestSelectNestedRecord.java | 71 ++++
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +-
.../rs/resources/TestQueryResultResource.java | 6 +-
.../TestSelectNestedRecord/sample1/table.json | 3 +
.../TestSelectNestedRecord/tweets/sample1.json | 4 +
.../TestSelectNestedRecord/sample1_ddl.sql | 7 +
.../TestSelectNestedRecord/sample2_ddl.sql | 19 ++
.../testNestedFieldAsGroupbyKey1.sql | 7 +
.../testNestedFieldAsJoinKey1.sql | 7 +
.../TestSelectNestedRecord/testSelect1.sql | 1 +
.../TestSelectNestedRecord/testSelect2.sql | 61 ++++
.../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++
.../testSelectWithParentheses2.sql | 2 +-
.../testNestedFieldAsGroupbyKey1.result | 6 +
.../testNestedFieldAsJoinKey1.result | 6 +
.../TestSelectNestedRecord/testSelect1.result | 5 +
.../TestSelectNestedRecord/testSelect2.result | 6 +
.../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +-
.../tajo/plan/LogicalPlanPreprocessor.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 32 +-
.../apache/tajo/plan/expr/RowConstantEval.java | 3 +-
.../tajo/plan/expr/WindowFunctionEval.java | 3 +-
.../GreedyHeuristicJoinOrderAlgorithm.java | 8 +-
.../apache/tajo/plan/joinorder/JoinEdge.java | 4 +-
.../apache/tajo/plan/joinorder/JoinGraph.java | 6 +-
.../tajo/plan/logical/DistinctGroupbyNode.java | 4 +-
.../apache/tajo/plan/logical/EvalExprNode.java | 3 +-
.../apache/tajo/plan/logical/GroupbyNode.java | 10 +-
.../tajo/plan/logical/ProjectionNode.java | 3 +-
.../apache/tajo/plan/logical/RelationNode.java | 12 +
.../tajo/plan/logical/ShuffleFileWriteNode.java | 3 +-
.../tajo/plan/logical/TruncateTableNode.java | 4 +-
.../apache/tajo/plan/logical/WindowAggNode.java | 7 +-
.../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++---
.../plan/nameresolver/ResolverByLegacy.java | 8 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 4 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 2 +-
.../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +-
.../plan/verifier/PreLogicalPlanVerifier.java | 6 +-
.../org/apache/tajo/rpc/AsyncRpcServer.java | 6 +-
.../org/apache/tajo/rpc/BlockingRpcServer.java | 6 +-
.../org/apache/tajo/storage/MergeScanner.java | 2 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 9 +
.../org/apache/tajo/storage/StorageManager.java | 4 +-
.../storage/hbase/AbstractHBaseAppender.java | 2 -
.../tajo/storage/hbase/ColumnMapping.java | 2 +-
.../HBaseBinarySerializerDeserializer.java | 10 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 22 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 6 +-
.../java/org/apache/tajo/storage/CSVFile.java | 2 +-
.../org/apache/tajo/storage/FileScanner.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 2 +-
.../apache/tajo/storage/avro/AvroScanner.java | 25 +-
.../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++--------
.../apache/tajo/storage/json/JsonLineSerDe.java | 5 +-
.../tajo/storage/json/JsonLineSerializer.java | 2 -
.../storage/parquet/TajoRecordConverter.java | 7 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 16 +-
.../tajo/storage/text/CSVLineDeserializer.java | 26 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 5 +-
.../tajo/storage/text/DelimitedTextFile.java | 11 +-
.../tajo/storage/text/TextLineDeserializer.java | 4 +-
.../apache/tajo/storage/text/TextLineSerDe.java | 3 +-
.../apache/tajo/storage/TestMergeScanner.java | 16 +-
.../org/apache/tajo/storage/TestStorages.java | 36 +-
99 files changed, 1292 insertions(+), 462 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 9788307,84be66a..73fd2f3
--- a/CHANGES
+++ b/CHANGES
@@@ -24,6 -24,6 +24,8 @@@ Release 0.11.0 - unrelease
IMPROVEMENT
++ TAJO-1591: Change StoreType represented as Enum to String type. (hyunsik)
++
TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun,
Committed by hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index e2cfe8f,322818d..2c856e1
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@@ -175,7 -175,19 +175,19 @@@ public class TestMergeScanner
Tuple tuple;
while ((tuple = scanner.next()) != null) {
totalCounts++;
- if (isProjectableStorage(meta.getStoreType())) {
+
- if (storeType == StoreType.RAW) {
++ if (storeType.equalsIgnoreCase("RAW")) {
+ assertEquals(4, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ assertNotNull(tuple.get(2));
+ assertNotNull(tuple.get(3));
+ } else if (scanner.isProjectable()) {
+ assertEquals(2, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ } else {
+ assertEquals(4, tuple.size());
assertNotNull(tuple.get(0));
assertNull(tuple.get(1));
assertNotNull(tuple.get(2));
@@@ -187,15 -199,15 +199,13 @@@
assertEquals(tupleNum * 2, totalCounts);
}
- private static boolean isProjectableStorage(StoreType type) {
- switch (type) {
- case CSV:
- case SEQUENCEFILE:
- case RAW:
- case ROWFILE:
- return false;
- default:
+ private static boolean isProjectableStorage(String type) {
+ if (type.equalsIgnoreCase("RCFILE") ||
+ type.equalsIgnoreCase("PARQUET") ||
- type.equalsIgnoreCase("SEQUENCEFILE") ||
- type.equalsIgnoreCase("CSV") ||
+ type.equalsIgnoreCase("AVRO")) {
return true;
+ } else {
+ return false;
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/53ed1c37/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --cc tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 7c75973,a735307..949e2e8
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@@ -313,9 -305,23 +305,23 @@@ public class TestStorages
assertEquals(tupleNum, tupleCnt);
}
+ private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCnt) {
+ if (projectable) {
+ assertTrue(tupleCnt + 2 == tuple.get(0).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(1).asFloat4());
+ } else {
+ // RAW and ROW always project all fields.
- if (storeType != StoreType.RAW && storeType != StoreType.ROWFILE) {
++ if (!storeType.equalsIgnoreCase("RAW") && !storeType.equalsIgnoreCase("ROWFILE")) {
+ assertTrue(tuple.get(0) == null);
+ }
+ assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
+ }
+ }
+
@Test
public void testVariousTypes() throws IOException {
- boolean handleProtobuf = storeType != StoreType.JSON;
+ boolean handleProtobuf = !storeType.equalsIgnoreCase("JSON");
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
@@@ -956,7 -962,7 +962,8 @@@
@Test
public void testLessThanSchemaSize() throws IOException {
/* RAW is internal storage. It must be same with schema size */
- if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")){
- if (storeType == StoreType.RAW || storeType == StoreType.AVRO || storeType == StoreType.PARQUET) {
++ if (storeType.equalsIgnoreCase("RAW") || storeType.equalsIgnoreCase("AVRO")
++ || storeType.equalsIgnoreCase("PARQUET")){
return;
}
[12/14] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
Posted by ji...@apache.org.
TAJO-1359: Add nested field projector and language extension to project nested record. (hyunsik)
Closes #422
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0d1bf41f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0d1bf41f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0d1bf41f
Branch: refs/heads/index_support
Commit: 0d1bf41f3aff119af5d95270d735101a81d5721c
Parents: ddd3921
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon May 11 14:34:07 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon May 11 14:35:09 2015 -0700
----------------------------------------------------------------------
CHANGES | 7 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 12 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++
.../java/org/apache/tajo/catalog/Schema.java | 109 +++++-
.../org/apache/tajo/catalog/SchemaUtil.java | 33 +-
.../tajo/catalog/store/HiveCatalogStore.java | 4 +-
.../cli/tsql/commands/DescTableCommand.java | 4 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Tuple.java | 2 +-
.../java/org/apache/tajo/util/StringUtils.java | 70 ++++
.../main/java/org/apache/tajo/util/TUtil.java | 32 --
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +-
.../engine/planner/PhysicalPlannerImpl.java | 17 +-
.../tajo/engine/planner/global/DataChannel.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 6 +-
.../global/builder/DistinctGroupbyBuilder.java | 6 +-
.../DistinctGroupbyThirdAggregationExec.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 27 +-
.../tajo/engine/utils/TupleCacheScanner.java | 2 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 8 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../NonForwardQueryResultSystemScanner.java | 20 +-
.../resources/webapps/admin/catalogview.jsp | 5 +-
.../engine/function/TestFunctionLoader.java | 5 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 8 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 4 +-
.../tajo/engine/query/TestJoinBroadcast.java | 2 +-
.../engine/query/TestSelectNestedRecord.java | 71 ++++
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +-
.../rs/resources/TestQueryResultResource.java | 6 +-
.../TestSelectNestedRecord/sample1/table.json | 3 +
.../TestSelectNestedRecord/tweets/sample1.json | 4 +
.../TestSelectNestedRecord/sample1_ddl.sql | 7 +
.../TestSelectNestedRecord/sample2_ddl.sql | 19 ++
.../testNestedFieldAsGroupbyKey1.sql | 7 +
.../testNestedFieldAsJoinKey1.sql | 7 +
.../TestSelectNestedRecord/testSelect1.sql | 1 +
.../TestSelectNestedRecord/testSelect2.sql | 61 ++++
.../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++
.../testSelectWithParentheses2.sql | 2 +-
.../testNestedFieldAsGroupbyKey1.result | 6 +
.../testNestedFieldAsJoinKey1.result | 6 +
.../TestSelectNestedRecord/testSelect1.result | 5 +
.../TestSelectNestedRecord/testSelect2.result | 6 +
.../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +-
.../tajo/plan/LogicalPlanPreprocessor.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 32 +-
.../apache/tajo/plan/expr/RowConstantEval.java | 3 +-
.../tajo/plan/expr/WindowFunctionEval.java | 3 +-
.../function/python/PythonScriptEngine.java | 12 +-
.../GreedyHeuristicJoinOrderAlgorithm.java | 8 +-
.../apache/tajo/plan/joinorder/JoinEdge.java | 4 +-
.../apache/tajo/plan/joinorder/JoinGraph.java | 6 +-
.../tajo/plan/logical/DistinctGroupbyNode.java | 4 +-
.../apache/tajo/plan/logical/EvalExprNode.java | 3 +-
.../apache/tajo/plan/logical/GroupbyNode.java | 10 +-
.../tajo/plan/logical/ProjectionNode.java | 3 +-
.../apache/tajo/plan/logical/RelationNode.java | 12 +
.../tajo/plan/logical/ShuffleFileWriteNode.java | 3 +-
.../tajo/plan/logical/TruncateTableNode.java | 4 +-
.../apache/tajo/plan/logical/WindowAggNode.java | 7 +-
.../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++---
.../plan/nameresolver/ResolverByLegacy.java | 8 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 4 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 2 +-
.../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +-
.../plan/verifier/PreLogicalPlanVerifier.java | 6 +-
.../org/apache/tajo/storage/MergeScanner.java | 2 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 9 +
.../org/apache/tajo/storage/StorageManager.java | 4 +-
.../storage/hbase/AbstractHBaseAppender.java | 2 -
.../tajo/storage/hbase/ColumnMapping.java | 2 +-
.../HBaseBinarySerializerDeserializer.java | 10 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 22 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 6 +-
.../java/org/apache/tajo/storage/CSVFile.java | 2 +-
.../org/apache/tajo/storage/FileScanner.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 2 +-
.../apache/tajo/storage/avro/AvroScanner.java | 25 +-
.../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++--------
.../apache/tajo/storage/json/JsonLineSerDe.java | 5 +-
.../tajo/storage/json/JsonLineSerializer.java | 2 -
.../storage/parquet/TajoRecordConverter.java | 7 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 16 +-
.../tajo/storage/text/CSVLineDeserializer.java | 26 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 5 +-
.../tajo/storage/text/DelimitedTextFile.java | 11 +-
.../tajo/storage/text/TextLineDeserializer.java | 4 +-
.../apache/tajo/storage/text/TextLineSerDe.java | 3 +-
.../apache/tajo/storage/TestMergeScanner.java | 29 +-
.../org/apache/tajo/storage/TestStorages.java | 35 +-
96 files changed, 1279 insertions(+), 467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0aad306..84be66a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,8 +27,8 @@ Release 0.11.0 - unreleased
TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun,
Committed by hyunsik)
- TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates
- an important kind of information.
+ TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage()
+ eliminates an important kind of information.
(Contributed by Jongyoung Park, Committed by jihoon)
TAJO-1408: Make IntermediateEntryProto more compact.
@@ -277,6 +277,9 @@ Release 0.11.0 - unreleased
SUB TASKS
+ TAJO-1359: Add nested field projector and language extension to project
+ nested record. (hyunsik)
+
TAJO-1529: Implement json_extract_path_text(string, string) function.
(jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 12f36b1..8e5b657 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -171,6 +171,16 @@ public class CatalogUtil {
return openQuote && closeQuote;
}
+ /**
+ * True if a given name is a simple identifier, meaning is not a dot-chained name.
+ *
+ * @param columnOrTableName Column or Table name to be checked
+ * @return True if a given name is a simple identifier. Otherwise, it will return False.
+ */
+ public static boolean isSimpleIdentifier(String columnOrTableName) {
+ return columnOrTableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 1;
+ }
+
public static boolean isFQColumnName(String tableName) {
return tableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 3;
}
@@ -662,7 +672,7 @@ public class CatalogUtil {
if (types[i].getType() != Type.NULL_TYPE) {
Type candidate = TUtil.getFromNestedMap(OPERATION_CASTING_MAP, widest.getType(), types[i].getType());
if (candidate == null) {
- throw new InvalidOperationException("No matched operation for those types: " + TUtil.arrayToString
+ throw new InvalidOperationException("No matched operation for those types: " + StringUtils.join
(types));
}
widest = newSimpleDataType(candidate);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
new file mode 100644
index 0000000..58b4f26
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.catalog;
+
+import com.google.common.base.Preconditions;
+import org.apache.tajo.common.TajoDataTypes.Type;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Utility methods for nested field
+ */
+public class NestedPathUtil {
+ public static final String PATH_DELIMITER = "/";
+
+ public static final List<String> ROOT_PATH = Collections.unmodifiableList(new ArrayList<String>());
+
+ public static boolean isPath(String name) {
+ return name.indexOf(PATH_DELIMITER.charAt(0)) >= 0;
+ }
+
+ public static String makePath(String[] parts) {
+ return makePath(parts, 0);
+ }
+
+ public static String makePath(String[] parts, int startIndex) {
+ return makePath(parts, startIndex, parts.length);
+ }
+
+ /**
+ * Make a nested field path
+ *
+ * @param parts path parts
+ * @param startIndex startIndex
+ * @param depth Depth
+ * @return Path
+ */
+ public static String makePath(String[] parts, int startIndex, int depth) {
+ Preconditions.checkArgument(startIndex <= (parts.length - 1));
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = startIndex; i < depth; i++) {
+ sb.append(PATH_DELIMITER);
+ sb.append(parts[i].toString());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Lookup the actual column corresponding to a given path.
+ * We assume that a path starts with the slash '/' and it
+ * does not include the root field.
+ *
+ * @param nestedField Nested column
+ * @param path Path which starts with '/';
+ * @return Column corresponding to the path
+ */
+ public static Column lookupPath(Column nestedField, String path) {
+ Preconditions.checkArgument(path.charAt(0) == PATH_DELIMITER.charAt(0),
+ "A nested field path must start with slash '/'.");
+
+ // We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
+ // So, we should start the index from 1 instead of 0.
+ return lookupPath(nestedField, path.split(PATH_DELIMITER));
+ }
+
+ public static Column lookupPath(Column nestedField, String [] paths) {
+ // We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
+ // So, we should start the index from 1 instead of 0.
+ return lookupColumnInternal(nestedField, paths, 1);
+ }
+
+ private static Column lookupColumnInternal(Column currentColumn, String [] paths, int depth) {
+ Column found = null;
+
+ if (currentColumn.getDataType().getType() == Type.RECORD) {
+ found = currentColumn.typeDesc.nestedRecordSchema.getColumn(paths[depth]);
+ }
+
+ if (found != null) {
+ if (found.getDataType().getType() == Type.RECORD) {
+ return lookupColumnInternal(found, paths, depth + 1);
+ } else {
+ return found;
+ }
+ } else {
+ throw new NoSuchFieldError(makePath(paths));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 054cc2c..0e4b741 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -32,6 +32,7 @@ import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.json.GsonObject;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -135,7 +136,8 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @param qualifier The qualifier
*/
public void setQualifier(String qualifier) {
- List<Column> columns = getColumns();
+ // only change root fields, and must keep each nested field simple name
+ List<Column> columns = getRootColumns();
fields.clear();
fieldsByQualifiedName.clear();
@@ -180,14 +182,39 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @return The column matched to a given column name.
*/
public Column getColumn(String name) {
- String [] parts = name.split("\\.");
- // Some of the string can includes database name and table name and column name.
- // For example, it can be 'default.table1.id'.
- // Therefore, spilt string array length can be 3.
- if (parts.length >= 2) {
- return getColumnByQName(name);
+
+ if (NestedPathUtil.isPath(name)) {
+
+ // TODO - to be refactored
+ if (fieldsByQualifiedName.containsKey(name)) {
+ Column flattenColumn = fields.get(fieldsByQualifiedName.get(name));
+ if (flattenColumn != null) {
+ return flattenColumn;
+ }
+ }
+
+ String [] paths = name.split(NestedPathUtil.PATH_DELIMITER);
+ Column column = getColumn(paths[0]);
+ if (column == null) {
+ return null;
+ }
+ Column actualColumn = NestedPathUtil.lookupPath(column, paths);
+
+ Column columnPath = new Column(
+ column.getQualifiedName() + NestedPathUtil.makePath(paths, 1),
+ actualColumn.typeDesc);
+
+ return columnPath;
} else {
- return getColumnByName(name);
+ String[] parts = name.split("\\.");
+ // Some of the string can includes database name and table name and column name.
+ // For example, it can be 'default.table1.id'.
+ // Therefore, spilt string array length can be 3.
+ if (parts.length >= 2) {
+ return getColumnByQName(name);
+ } else {
+ return getColumnByName(name);
+ }
}
}
@@ -268,12 +295,46 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
return -1;
}
-
- public List<Column> getColumns() {
+
+ /**
+ * Get root columns, meaning all columns except for nested fields.
+ *
+ * @return A list of root columns
+ */
+ public List<Column> getRootColumns() {
return ImmutableList.copyOf(fields);
}
+ /**
+ * Get all columns, including all nested fields
+ *
+ * @return A list of all columns
+ */
+ public List<Column> getAllColumns() {
+ final List<Column> columnList = TUtil.newList();
+
+ SchemaUtil.visitSchema(this, new ColumnVisitor() {
+ @Override
+ public void visit(int depth, List<String> path, Column column) {
+ if (path.size() > 0) {
+ String parentPath = StringUtils.join(path, NestedPathUtil.PATH_DELIMITER);
+ String currentPath = parentPath + NestedPathUtil.PATH_DELIMITER + column.getSimpleName();
+ columnList.add(new Column(currentPath, column.getTypeDesc()));
+ } else {
+ columnList.add(column);
+ }
+ }
+ });
+
+ return columnList;
+ }
+
public boolean contains(String name) {
+ // TODO - It's a hack
+ if (NestedPathUtil.isPath(name)) {
+ return (getColumn(name) != null);
+ }
+
if (fieldsByQualifiedName.containsKey(name)) {
return true;
}
@@ -288,6 +349,11 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public boolean contains(Column column) {
+ // TODO - It's a hack
+ if (NestedPathUtil.isPath(column.getQualifiedName())) {
+ return (getColumn(column.getQualifiedName()) != null);
+ }
+
if (column.hasQualifier()) {
return fieldsByQualifiedName.containsKey(column.getQualifiedName());
} else {
@@ -314,7 +380,24 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public boolean containsAll(Collection<Column> columns) {
- return fields.containsAll(columns);
+ boolean containFlag = true;
+
+ for (Column c :columns) {
+ if (NestedPathUtil.isPath(c.getSimpleName())) {
+ if (contains(c.getQualifiedName())) {
+ containFlag &= true;
+ } else {
+ String[] paths = c.getQualifiedName().split("/");
+ boolean existRootPath = contains(paths[0]);
+ boolean existLeafPath = getColumn(c.getSimpleName()) != null;
+ containFlag &= existRootPath && existLeafPath;
+ }
+ } else {
+ containFlag &= fields.contains(c);
+ }
+ }
+
+ return containFlag;
}
public synchronized Schema addColumn(String name, TypeDesc typeDesc) {
@@ -351,7 +434,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public synchronized void addColumns(Schema schema) {
- for(Column column : schema.getColumns()) {
+ for(Column column : schema.getRootColumns()) {
addColumn(column);
}
}
@@ -396,7 +479,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
@Override
- public void visit(int depth, Column column) {
+ public void visit(int depth, List<String> path, Column column) {
if (column.getDataType().getType() == Type.RECORD) {
DataType.Builder updatedType = DataType.newBuilder(column.getDataType());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
index f2bb71c..c6b2f69 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
@@ -18,6 +18,10 @@
package org.apache.tajo.catalog;
+import org.apache.tajo.util.TUtil;
+
+import java.util.List;
+
import static org.apache.tajo.common.TajoDataTypes.DataType;
import static org.apache.tajo.common.TajoDataTypes.Type;
@@ -34,12 +38,12 @@ public class SchemaUtil {
static int tmpColumnSeq = 0;
public static Schema merge(Schema left, Schema right) {
Schema merged = new Schema();
- for(Column col : left.getColumns()) {
+ for(Column col : left.getRootColumns()) {
if (!merged.containsByQualifiedName(col.getQualifiedName())) {
merged.addColumn(col);
}
}
- for(Column col : right.getColumns()) {
+ for(Column col : right.getRootColumns()) {
if (merged.containsByQualifiedName(col.getQualifiedName())) {
merged.addColumn("?fake" + (tmpColumnSeq++), col.getDataType());
} else {
@@ -59,7 +63,7 @@ public class SchemaUtil {
*/
public static Schema getNaturalJoinColumns(Schema left, Schema right) {
Schema common = new Schema();
- for (Column outer : left.getColumns()) {
+ for (Column outer : left.getRootColumns()) {
if (!common.containsByName(outer.getSimpleName()) && right.containsByName(outer.getSimpleName())) {
common.addColumn(new Column(outer.getSimpleName(), outer.getDataType()));
}
@@ -113,7 +117,7 @@ public class SchemaUtil {
* Column visitor interface
*/
public static interface ColumnVisitor {
- public void visit(int depth, Column column);
+ public void visit(int depth, List<String> path, Column column);
}
/**
@@ -122,8 +126,8 @@ public class SchemaUtil {
* @param function
*/
public static void visitSchema(Schema schema, ColumnVisitor function) {
- for(Column col : schema.getColumns()) {
- visitInDepthFirstOrder(0, function, col);
+ for(Column col : schema.getRootColumns()) {
+ visitInDepthFirstOrder(0, NestedPathUtil.ROOT_PATH, function, col);
}
}
@@ -134,14 +138,21 @@ public class SchemaUtil {
* @param function Visitor
* @param column Current visiting column
*/
- private static void visitInDepthFirstOrder(int depth, ColumnVisitor function, Column column) {
+ private static void visitInDepthFirstOrder(int depth,
+ final List<String> path,
+ ColumnVisitor function,
+ Column column) {
+
if (column.getDataType().getType() == Type.RECORD) {
- for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getColumns()) {
- visitInDepthFirstOrder(depth + 1, function, nestedColumn);
+ for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getRootColumns()) {
+ List<String> newPath = TUtil.newList(path);
+ newPath.add(column.getQualifiedName());
+
+ visitInDepthFirstOrder(depth + 1, newPath, function, nestedColumn);
}
- function.visit(depth, column);
+ function.visit(depth, path, column);
} else {
- function.visit(depth, column);
+ function.visit(depth, path, column);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index 5b1a996..6acdbd1 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -453,7 +453,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
}
// set column information
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size());
for (Column eachField : columns) {
@@ -465,7 +465,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
// set partition keys
if (tableDesc.hasPartition() && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>();
- for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), ""));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index a3960e6..a5b53f2 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@ -27,7 +27,7 @@ import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.cli.tsql.TajoCli;
import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.List;
import java.util.Map;
@@ -126,7 +126,7 @@ public class DescTableCommand extends TajoShellCommand {
sb.append("type:").append(partition.getPartitionType().name()).append("\n");
sb.append("columns:").append(":");
- sb.append(TUtil.arrayToString(partition.getExpressionSchema().toArray()));
+ sb.append(StringUtils.join(partition.getExpressionSchema().toArray()));
}
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 385f99c..6e16095 100644
--- a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -36,7 +36,7 @@ public class RowStoreUtil {
public static int[] getTargetIds(Schema inSchema, Schema outSchema) {
int[] targetIds = new int[outSchema.size()];
int i = 0;
- for (Column target : outSchema.getColumns()) {
+ for (Column target : outSchema.getRootColumns()) {
targetIds[i] = inSchema.getColumnId(target.getQualifiedName());
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
index aec784f..fce10a3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
@@ -41,8 +41,8 @@ public interface Tuple extends Cloneable {
void put(Datum[] values);
+
Datum get(int fieldId);
-
void setOffset(long offset);
long getOffset();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index d035e4a..0a16072 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -371,4 +371,74 @@ public class StringUtils {
return resultArray;
}
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects Iterable objects
+ * @param delimiter Delimiter string
+ * @return A joined string
+ */
+ public static String join(Iterable objects, String delimiter) {
+ boolean first = true;
+ StringBuilder sb = new StringBuilder();
+ for(Object object : objects) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(delimiter);
+ }
+
+ sb.append(object.toString());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Concatenate all objects' string with the delimiter ", "
+ *
+ * @param objects Iterable objects
+ * @return A joined string
+ */
+ public static String join(Object[] objects) {
+ return join(objects, ", ", 0, objects.length);
+ }
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param startIndex the begin index to join
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter, int startIndex) {
+ return join(objects, delimiter, startIndex, objects.length);
+ }
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param startIndex the begin index to join
+ * @param length the number of columns to be joined
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter, int startIndex, int length) {
+ boolean first = true;
+ StringBuilder sb = new StringBuilder();
+ for(int i = startIndex; i + startIndex < length; i++) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(delimiter);
+ }
+
+ sb.append(objects[i].toString());
+ }
+
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
index 75f3e2a..2293ef5 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
@@ -240,38 +240,6 @@ public class TUtil {
}
}
- public static String collectionToString(Collection objects, String delimiter) {
- boolean first = true;
- StringBuilder sb = new StringBuilder();
- for(Object object : objects) {
- if (first) {
- first = false;
- } else {
- sb.append(delimiter);
- }
-
- sb.append(object.toString());
- }
-
- return sb.toString();
- }
-
- public static String arrayToString(Object [] objects) {
- boolean first = true;
- StringBuilder sb = new StringBuilder();
- for(Object object : objects) {
- if (first) {
- first = false;
- } else {
- sb.append(", ");
- }
-
- sb.append(object.toString());
- }
-
- return sb.toString();
- }
-
public static <T> T [] toArray(Collection<T> collection, Class<T> type) {
T array = (T) Array.newInstance(type, collection.size());
return collection.toArray((T[]) array);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 1a63470..3ab11bd 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -1307,7 +1307,7 @@ set_qualifier
;
column_reference
- : ((db_name = identifier DOT)? (tb_name=identifier DOT))? name=identifier
+ : identifier (DOT identifier)*
;
as_clause
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index 6d32fa5..7c99868 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1054,14 +1054,17 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
@Override
public ColumnReferenceExpr visitColumn_reference(SQLParser.Column_referenceContext ctx) {
- ColumnReferenceExpr column = new ColumnReferenceExpr(ctx.name.getText());
- if (checkIfExist(ctx.db_name)) {
- column.setQualifier(CatalogUtil.buildFQName(ctx.db_name.getText(), ctx.tb_name.getText()));
- } else if (ctx.tb_name != null) {
- column.setQualifier(ctx.tb_name.getText());
+ String columnReferenceName = ctx.getText();
+ // find the last dot (.) position to separate a name into both a qualifier and name
+ int lastDotIdx = columnReferenceName.lastIndexOf(".");
+
+ if (lastDotIdx > 0) { // if any qualifier is given
+ String qualifier = columnReferenceName.substring(0, lastDotIdx);
+ String name = columnReferenceName.substring(lastDotIdx + 1, columnReferenceName.length());
+ return new ColumnReferenceExpr(qualifier, name);
+ } else {
+ return new ColumnReferenceExpr(ctx.getText());
}
-
- return column;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index f132793..506b03e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -55,6 +55,7 @@ import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.IndexUtil;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -275,7 +276,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.",
context.getTaskId().toString(),
(left ? "Left" : "Right"),
- TUtil.arrayToString(lineage),
+ StringUtils.join(lineage),
FileUtil.humanReadableByteCount(volume, false),
(inMemoryInnerJoinFlag ? "" : "not ")));
return inMemoryInnerJoinFlag;
@@ -398,18 +399,18 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
larger = right;
LOG.info(String.format("[%s] Left relations %s (%s) is smaller than Right relations %s (%s).",
context.getTaskId().toString(),
- TUtil.arrayToString(leftLineage),
+ StringUtils.join(leftLineage),
FileUtil.humanReadableByteCount(leftSize, false),
- TUtil.arrayToString(rightLineage),
+ StringUtils.join(rightLineage),
FileUtil.humanReadableByteCount(rightSize, false)));
} else {
smaller = right;
larger = left;
LOG.info(String.format("[%s] Right relations %s (%s) is smaller than Left relations %s (%s).",
context.getTaskId().toString(),
- TUtil.arrayToString(rightLineage),
+ StringUtils.join(rightLineage),
FileUtil.humanReadableByteCount(rightSize, false),
- TUtil.arrayToString(leftLineage),
+ StringUtils.join(leftLineage),
FileUtil.humanReadableByteCount(leftSize, false)));
}
@@ -858,7 +859,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
} else if (storeTableNode.getType() == NodeType.CREATE_TABLE) {
int i = 0;
for (int j = 0; j < partitionKeyColumns.length; j++) {
- int id = storeTableNode.getOutSchema().getColumns().size() + j;
+ int id = storeTableNode.getOutSchema().getRootColumns().size() + j;
Column column = storeTableNode.getInSchema().getColumn(id);
sortSpecs[i++] = new SortSpec(column, true, false);
}
@@ -1002,7 +1003,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
sortNode.setInSchema(subOp.getSchema());
sortNode.setOutSchema(subOp.getSchema());
ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp);
- LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")");
+ LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")");
return new SortAggregateExec(ctx, groupbyNode, sortExec);
}
@@ -1043,7 +1044,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
sortNode.setInSchema(subOp.getSchema());
sortNode.setOutSchema(subOp.getSchema());
child = new ExternalSortExec(context, sortNode, subOp);
- LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")");
+ LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")");
}
return new WindowAggExec(context, windowAggNode, child);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
index 11548d3..ba1b0bf 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
@@ -23,7 +23,7 @@ import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
@@ -193,7 +193,7 @@ public class DataChannel {
sb.append(" (type=").append(shuffleType);
if (hasShuffleKeys()) {
sb.append(", key=");
- sb.append(TUtil.arrayToString(shuffleKeys));
+ sb.append(StringUtils.join(shuffleKeys));
sb.append(", num=").append(numOutputs);
}
sb.append(")");
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 36bdf21..ff1955e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -1146,11 +1146,11 @@ public class GlobalPlanner {
Column[] shuffleKeys = new Column[partitionMethod.getExpressionSchema().size()];
int i = 0, id = 0;
- for (Column column : partitionMethod.getExpressionSchema().getColumns()) {
+ for (Column column : partitionMethod.getExpressionSchema().getRootColumns()) {
if (node.getType() == NodeType.INSERT) {
id = tableSchema.getColumnId(column.getQualifiedName());
} else {
- id = tableSchema.getColumns().size() + i;
+ id = tableSchema.getRootColumns().size() + i;
}
shuffleKeys[i++] = projectedSchema.getColumn(id);
}
@@ -1493,7 +1493,7 @@ public class GlobalPlanner {
addedTableSubQueries.add(copy);
//Find a SubQueryNode which contains all columns in InputSchema matched with Target and OutputSchema's column
- if (copy.getInSchema().containsAll(copy.getOutSchema().getColumns())) {
+ if (copy.getInSchema().containsAll(copy.getOutSchema().getRootColumns())) {
for (Target eachTarget : copy.getTargets()) {
Set<Column> columns = EvalTreeUtil.findUniqueColumns(eachTarget.getEvalTree());
if (copy.getInSchema().containsAll(columns)) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
index b5e9104..13ed99b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
@@ -81,13 +81,13 @@ public class DistinctGroupbyBuilder {
// If there is not grouping column, we can't find column alias.
// Thus we should find the alias at Groupbynode output schema.
if (groupbyNode.getGroupingColumns().length == 0
- && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) {
+ && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) {
aggFunctions[i].setAlias(groupbyNode.getOutSchema().getColumn(i).getQualifiedName());
}
}
if (groupbyNode.getGroupingColumns().length == 0
- && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) {
+ && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) {
groupbyNode.setAggFunctions(aggFunctions);
}
@@ -672,7 +672,7 @@ public class DistinctGroupbyBuilder {
int index = 0;
for(GroupbyNode eachNode: secondStageDistinctNode.getSubPlans()) {
eachNode.setInSchema(firstStageDistinctNode.getOutSchema());
- for (Column column: eachNode.getOutSchema().getColumns()) {
+ for (Column column: eachNode.getOutSchema().getRootColumns()) {
if (secondStageInSchema.getColumn(column) == null) {
secondStageInSchema.addColumn(column);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
index e71976c..7c38d36 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
@@ -110,7 +110,7 @@ public class DistinctGroupbyThirdAggregationExec extends UnaryPhysicalExec {
}
int index = 0;
- for (Column eachOutputColumn: outSchema.getColumns()) {
+ for (Column eachOutputColumn: outSchema.getRootColumns()) {
// If column is avg aggregation function, outschema's column type is float
// but groupbyResultTupleIndex's column type is protobuf
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
index 247b373..7694b2b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
@@ -77,7 +77,7 @@ public class PhysicalPlanUtil {
//In the case of partitioned table, we should return same partition key data files.
int partitionDepth = 0;
if (tableDesc.hasPartition()) {
- partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size();
+ partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size();
}
List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
index ff9477f..3d95068 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
@@ -138,6 +138,8 @@ public class SeqScanExec extends ScanExec {
public void init() throws IOException {
Schema projected;
+ // in the case where projected column or expression are given
+ // the target can be an empty list.
if (plan.hasTargets()) {
projected = new Schema();
Set<Column> columnSet = new HashSet<Column>();
@@ -150,12 +152,15 @@ public class SeqScanExec extends ScanExec {
columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree()));
}
- for (Column column : inSchema.getColumns()) {
+ for (Column column : inSchema.getAllColumns()) {
if (columnSet.contains(column)) {
projected.addColumn(column);
}
}
+
} else {
+ // no any projected columns, meaning that all columns should be projected.
+ // TODO - this implicit rule makes code readability bad. So, we should remove it later
projected = outSchema;
}
@@ -163,7 +168,11 @@ public class SeqScanExec extends ScanExec {
super.init();
if (plan.hasQual()) {
- qual.bind(context.getEvalContext(), inSchema);
+ if (scanner.isProjectable()) {
+ qual.bind(context.getEvalContext(), projected);
+ } else {
+ qual.bind(context.getEvalContext(), inSchema);
+ }
}
}
@@ -175,7 +184,7 @@ public class SeqScanExec extends ScanExec {
}
private void initScanner(Schema projected) throws IOException {
- this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
+
TableMeta meta;
try {
meta = (TableMeta) plan.getTableDesc().getMeta().clone();
@@ -186,6 +195,7 @@ public class SeqScanExec extends ScanExec {
// set system default properties
PlannerUtil.applySystemDefaultToTableProperties(context.getQueryContext(), meta);
+ // Why we should check nullity? See https://issues.apache.org/jira/browse/TAJO-1422
if (fragments != null) {
if (fragments.length > 1) {
this.scanner = new MergeScanner(context.getConf(), plan.getPhysicalSchema(), meta,
@@ -198,6 +208,17 @@ public class SeqScanExec extends ScanExec {
plan.getPhysicalSchema(), fragments[0], projected);
}
scanner.init();
+
+ // See Scanner.isProjectable() method Depending on the result of isProjectable(),
+ // the width of retrieved tuple is changed.
+ //
+ // If TRUE, the retrieved tuple will contain only projected fields.
+ // If FALSE, the retrieved tuple will contain projected fields and NullDatum for non-projected fields.
+ if (scanner.isProjectable()) {
+ this.projector = new Projector(context, projected, outSchema, plan.getTargets());
+ } else {
+ this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
index ba25172..0fd2fbe 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
@@ -73,7 +73,7 @@ public class TupleCacheScanner implements Scanner {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
index 3a0a1c7..c01900a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
@@ -89,7 +89,7 @@ public class TupleUtil {
}
int i = 0;
- for (Column col : sortSchema.getColumns()) {
+ for (Column col : sortSchema.getRootColumns()) {
ColumnStats columnStat = statMap.get(col);
if (columnStat == null) {
continue;
@@ -121,7 +121,7 @@ public class TupleUtil {
statSet.put(stat.getColumn(), stat);
}
- for (Column col : target.getColumns()) {
+ for (Column col : target.getRootColumns()) {
Preconditions.checkState(statSet.containsKey(col),
"ERROR: Invalid Column Stats (column stats: " + colStats + ", there exists not target " + col);
}
@@ -134,7 +134,7 @@ public class TupleUtil {
// In outer join, empty table could be searched.
// As a result, min value and max value would be null.
// So, we should put NullDatum for this case.
- for (Column col : target.getColumns()) {
+ for (Column col : target.getRootColumns()) {
if (sortSpecs[sortSpecIndex].isAscending()) {
if (statSet.get(col).getMinValue() != null)
startTuple.put(i, statSet.get(col).getMinValue());
@@ -169,7 +169,7 @@ public class TupleUtil {
else
endTuple.put(i, DatumFactory.createNullDatum());
}
- if (target.getColumns().size() == sortSpecs.length) {
+ if (target.getRootColumns().size() == sortSpecs.length) {
// Not composite column sort
sortSpecIndex++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
index 804821b..9c0bd48 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
@@ -89,7 +89,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc
StringBuffer path = new StringBuffer();
int depth = 0;
if (tableDesc.hasPartition()) {
- for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
String partitionValue = EvalTreeUtil.getPartitionValue(scanNode.getQual(), c.getSimpleName());
if (partitionValue == null)
break;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
index 958c252..6c9b485 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
@@ -144,7 +144,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getTablespaces(Schema outSchema) {
List<TablespaceProto> tablespaces = masterContext.getCatalog().getAllTablespaces();
List<Tuple> tuples = new ArrayList<Tuple>(tablespaces.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablespaceProto tablespace: tablespaces) {
@@ -179,7 +179,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getDatabases(Schema outSchema) {
List<DatabaseProto> databases = masterContext.getCatalog().getAllDatabases();
List<Tuple> tuples = new ArrayList<Tuple>(databases.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (DatabaseProto database: databases) {
@@ -209,7 +209,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getTables(Schema outSchema) {
List<TableDescriptorProto> tables = masterContext.getCatalog().getAllTables();
List<Tuple> tuples = new ArrayList<Tuple>(tables.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableDescriptorProto table: tables) {
@@ -245,7 +245,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getColumns(Schema outSchema) {
List<ColumnProto> columnsList = masterContext.getCatalog().getAllColumns();
List<Tuple> tuples = new ArrayList<Tuple>(columnsList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
int columnId = 1, prevtid = -1, tid = 0;
@@ -293,7 +293,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getIndexes(Schema outSchema) {
List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes();
List<Tuple> tuples = new ArrayList<Tuple>(indexList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (IndexProto index: indexList) {
@@ -332,7 +332,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllTableOptions(Schema outSchema) {
List<TableOptionProto> optionList = masterContext.getCatalog().getAllTableOptions();
List<Tuple> tuples = new ArrayList<Tuple>(optionList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableOptionProto option: optionList) {
@@ -359,7 +359,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllTableStats(Schema outSchema) {
List<TableStatsProto> statList = masterContext.getCatalog().getAllTableStats();
List<Tuple> tuples = new ArrayList<Tuple>(statList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableStatsProto stat: statList) {
@@ -386,7 +386,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllPartitions(Schema outSchema) {
List<TablePartitionProto> partitionList = masterContext.getCatalog().getAllPartitions();
List<Tuple> tuples = new ArrayList<Tuple>(partitionList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablePartitionProto partition: partitionList) {
@@ -417,7 +417,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
}
private Tuple getQueryMasterTuple(Schema outSchema, Worker aWorker) {
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
WorkerResource aResource = aWorker.getResource();
@@ -463,7 +463,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
}
private Tuple getWorkerTuple(Schema outSchema, Worker aWorker) {
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
WorkerResource aResource = aWorker.getResource();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
index 1ff81a6..e014379 100644
--- a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
@@ -30,7 +30,6 @@
<%@ page import="java.util.Collection" %>
<%@ page import="java.util.List" %>
<%@ page import="java.util.Map" %>
-<%@ page import="org.apache.tajo.service.ServiceTracker" %>
<%
TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
CatalogService catalog = master.getCatalog();
@@ -143,7 +142,7 @@
<div style='margin-top:5px'>
<%
if(tableDesc != null) {
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
out.write("<table border='1' class='border_table'><tr><th>No</th><th>Column name</th><th>Type</th></tr>");
int columnIndex = 1;
for(Column eachColumn: columns) {
@@ -155,7 +154,7 @@
if (tableDesc.getPartitionMethod() != null) {
PartitionMethodDesc partition = tableDesc.getPartitionMethod();
- List<Column> partitionColumns = partition.getExpressionSchema().getColumns();
+ List<Column> partitionColumns = partition.getExpressionSchema().getRootColumns();
String partitionColumnStr = "";
String prefix = "";
for (Column eachColumn: partitionColumns) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
index 32b98dd..cf34c33 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
@@ -20,8 +20,7 @@ package org.apache.tajo.engine.function;
import com.google.common.collect.Lists;
import org.apache.tajo.catalog.FunctionDesc;
-import org.apache.tajo.function.FunctionSignature;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import org.junit.Test;
import java.io.IOException;
@@ -38,7 +37,7 @@ public class TestFunctionLoader {
public void testFindScalarFunctions() throws IOException {
List<FunctionDesc> collections = Lists.newArrayList(FunctionLoader.findScalarFunctions());
Collections.sort(collections);
- String functionList = TUtil.collectionToString(collections, "\n");
+ String functionList = StringUtils.join(collections, "\n");
String result = getResultText(TestFunctionLoader.class, "testFindScalarFunctions.result");
assertEquals(result.trim(), functionList.trim());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index af0aa6a..dfac53d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -871,7 +871,7 @@ public class TestLogicalPlanner {
assertEquals(NodeType.EXPRS, root.getChild().getType());
Schema out = root.getOutSchema();
- Iterator<Column> it = out.getColumns().iterator();
+ Iterator<Column> it = out.getRootColumns().iterator();
Column col = it.next();
assertEquals("res1", col.getSimpleName());
col = it.next();
@@ -920,7 +920,7 @@ public class TestLogicalPlanner {
testJsonSerDerObject(root);
Schema finalSchema = root.getOutSchema();
- Iterator<Column> it = finalSchema.getColumns().iterator();
+ Iterator<Column> it = finalSchema.getRootColumns().iterator();
Column col = it.next();
assertEquals("deptname", col.getSimpleName());
col = it.next();
@@ -931,7 +931,7 @@ public class TestLogicalPlanner {
root = (LogicalRootNode) plan;
finalSchema = root.getOutSchema();
- it = finalSchema.getColumns().iterator();
+ it = finalSchema.getRootColumns().iterator();
col = it.next();
assertEquals("id", col.getSimpleName());
col = it.next();
@@ -948,7 +948,7 @@ public class TestLogicalPlanner {
testJsonSerDerObject(root);
Schema finalSchema = root.getOutSchema();
- Iterator<Column> it = finalSchema.getColumns().iterator();
+ Iterator<Column> it = finalSchema.getRootColumns().iterator();
Column col = it.next();
assertEquals("id", col.getSimpleName());
col = it.next();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index f79f703..727b1a2 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -68,7 +68,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertTrue(desc.getSchema().contains("default.testctaswithouttabledefinition.col1"));
PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
- assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
+ assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration());
Path path = new Path(desc.getPath());
@@ -111,7 +111,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
- assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
+ assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(cluster.getConfiguration());
Path path = new Path(desc.getPath());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
index 48aea26..a1eceea 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
@@ -707,7 +707,7 @@ public class TestJoinBroadcast extends QueryTestCaseBase {
"select a.o_orderdate, a.o_orderstatus, a.o_orderkey_mod, a.o_totalprice " +
"from " + tableName +
" a join "+ tableName + " b on a.o_orderkey = b.o_orderkey " +
- "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and o_orderkey_mod = 1 " +
+ "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and a.o_orderkey_mod = 1 " +
" order by a.o_orderkey"
);
String resultSetData = resultSetToString(res);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
new file mode 100644
index 0000000..9f8a5fd
--- /dev/null
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.util.TUtil;
+import org.junit.Test;
+
+import java.sql.ResultSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestSelectNestedRecord extends QueryTestCaseBase {
+
+ @Test
+ public final void testSelect1() throws Exception {
+ List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample1");
+ assertEquals(TUtil.newList("sample1"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testSelect2() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testNestedFieldAsGroupbyKey1() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testNestedFieldAsJoinKey1() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
index 0dbb8e5..c8c24cd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
@@ -262,7 +262,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
TableDesc tableDesc = client.getTableDesc(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
assertNotNull(tableDesc);
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
while (rs.next()) {
assertEquals(tableName, rs.getString("TABLE_NAME"));
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
index 6fc4ea1..1b23966 100644
--- a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
+++ b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
@@ -140,7 +140,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
@@ -174,7 +174,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
@@ -236,7 +236,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
new file mode 100644
index 0000000..db3ad6c
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
@@ -0,0 +1,3 @@
+{ "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}}
+{ "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}}
+{ "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
new file mode 100644
index 0000000..78a9071
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
@@ -0,0 +1,4 @@
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Mon Sep 24 03:35:21 +0000 2012","id_str":"250075927172759552","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Aggressive Ponytail #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":1,"in_reply_to_status_id_str":null,"id":250075927172759552,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDEEF6","profile_sidebar_border_color":"C0DEED","profile_background_tile":false,"name":"Sean Cummings","profile_image_url":"http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","created_at":"Mon Apr 26 06:01:55 +0000 2010","location":"LA, CA","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"137238150","entities":{"url":{"urls":[{"expanded_url":null,"url":"","i
ndices":[0,0]}]},"description":{"urls":[]}},"default_profile":true,"contributors_enabled":false,"favourites_count":0,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","utc_offset":-28800,"id":137238150,"profile_use_background_image":true,"listed_count":2,"profile_text_color":"333333","lang":"en","followers_count":70,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_background_color":"C0DEED","verified":false,"geo_enabled":true,"time_zone":"Pacific Time (US & Canada)","description":"Born 330 Live 310","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme1/bg.png","statuses_count":579,"friends_count":110,"following":null,"show_all_inline_media":false,"screen_name":"sean_cummings"},"in_reply_to_screen_name":null,"source":"<a>Twitter for Mac<\/a>","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:40:54 +0000 2012","id_str":"249292149810667520","entities":{"urls":[],"hashtags":[{"text":"FreeBandNames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Thee Namaste Nerdz. #FreeBandNames","metadata":{"iso_language_code":"pl","result_type":"recent"},"retweet_count":2,"in_reply_to_status_id_str":null,"id":249292149810667520,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDFFCC","profile_sidebar_border_color":"BDDCAD","profile_background_tile":true,"name":"Chaz Martenstein","profile_image_url":"http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","created_at":"Tue Apr 07 19:05:07 +0000 2009","location":"Durham, NC","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"29516238","entities":{"url":{"urls":[{"expanded_url":null,"url":"http://bu
llcityrecords.com/wnng/","indices":[0,32]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":8,"url":"http://bullcityrecords.com/wnng/","profile_image_url_https":"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","utc_offset":-18000,"id":29516238,"profile_use_background_image":true,"listed_count":118,"profile_text_color":"333333","lang":"en","followers_count":2052,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp","profile_background_color":"9AE4E8","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp","statuses_c
ount":7579,"friends_count":348,"following":null,"show_all_inline_media":true,"screen_name":"bullcityrecords"},"in_reply_to_screen_name":null,"source":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:30:20 +0000 2012","id_str":"249289491129438208","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[29,43]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Mexican Heaven, Mexican Hell #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":3,"in_reply_to_status_id_str":null,"id":249289491129438208,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"99CC33","profile_sidebar_border_color":"829D5E","profile_background_tile":false,"name":"Thomas John Wakeman","profile_image_url":"http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","created_at":"Tue Sep 01 21:21:35 +0000 2009","location":"Kingston New York","follow_request_sent":null,"profile_link_color":"D02B55","is_translator":false,"id_str":"70789458","entities":{"url":{"urls":[{"expanded_url":n
ull,"url":"","indices":[0,0]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":19,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","utc_offset":-18000,"id":70789458,"profile_use_background_image":true,"listed_count":1,"profile_text_color":"3E4415","lang":"en","followers_count":63,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_background_color":"352726","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme5/bg.gif","statuses_count":1048,"friends_count":63,"following":null,"show_all_inline_media":false,"screen_name":"MonkiesFist"},"in_reply_to_screen_name":null,"sour
ce":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 22:51:18 +0000 2012","id_str":"249279667666817024","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"The Foolish Mortals #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":4,"in_reply_to_status_id_str":null,"id":249279667666817024,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"BFAC83","profile_sidebar_border_color":"615A44","profile_background_tile":true,"name":"Marty Elmer","profile_image_url":"http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","created_at":"Mon May 04 00:05:00 +0000 2009","location":"Wisconsin, USA","follow_request_sent":null,"profile_link_color":"3B2A26","is_translator":false,"id_str":"37539828","entities":{"url":{"urls":[{"expanded_url":null,"url":"ht
tp://www.omnitarian.me","indices":[0,24]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":647,"url":"http://www.omnitarian.me","profile_image_url_https":"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","utc_offset":-21600,"id":37539828,"profile_use_background_image":true,"listed_count":52,"profile_text_color":"000000","lang":"en","followers_count":608,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png","profile_background_color":"EEE3C4","verified":false,"geo_enabled":false,"time_zone":"Central Time (US & Canada)","description":"Cartoonist, Illustrator, and T-Shirt connoisseur","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png","statuses_count":3575,"friends_count":249,"following":null,"show_all_inline_media":true,"scree
n_name":"Omnitarian"},"in_reply_to_screen_name":null,"source":"<a>Twitter for iPhone<\/a>","in_reply_to_status_id":null}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
new file mode 100644
index 0000000..9ba5f8c
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
@@ -0,0 +1,7 @@
+CREATE EXTERNAL TABLE ${0} (
+ title TEXT,
+ name RECORD (
+ first_name TEXT,
+ last_name TEXT
+ )
+) USING JSON LOCATION ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
new file mode 100644
index 0000000..9537c3e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
@@ -0,0 +1,19 @@
+CREATE EXTERNAL TABLE ${0} (
+ glossary RECORD (
+ title TEXT,
+ "GlossDiv" RECORD (
+ "GlossEntry" RECORD (
+ "ID" TEXT,
+ "SortAs" TEXT,
+ "GlossTerm" TEXT,
+ "Acronym" TEXT,
+ "Abbrev" TEXT,
+ "GlossDef" RECORD (
+ para TEXT
+ ),
+
+ "GlossSee" TEXT
+ )
+ )
+ )
+) USING JSON LOCATION ${path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
new file mode 100644
index 0000000..057ba05
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
@@ -0,0 +1,7 @@
+SELECT
+ user.name,
+ sum(retweet_count) as total_retweet
+FROM
+ tweets
+GROUP BY
+ user.name;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
new file mode 100644
index 0000000..336840e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
@@ -0,0 +1,7 @@
+SELECT
+ t1.user.id,
+ t1.user.name,
+ t2.user.id,
+ t2.user.name
+FROM
+ tweets t1 join tweets t2 ON t1.user.id = t2.user.id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
new file mode 100644
index 0000000..b099e77
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
@@ -0,0 +1 @@
+SELECT title, (name.first_name || ' ' || name.last_name) as full_name FROM sample1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
new file mode 100644
index 0000000..9993a4e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
@@ -0,0 +1,61 @@
+SELECT
+ coordinates,
+ favorited,
+ truncated,
+ created_at,
+ id_str,
+ in_reply_to_user_id_str,
+ contributors,
+ "text",
+ metadata.iso_language_code,
+ metadata.result_type,
+ retweet_count,
+ in_reply_to_status_id_str,
+ id,
+ geo,
+ retweeted,
+ in_reply_to_user_id,
+ place,
+ user.profile_sidebar_fill_color,
+ user.profile_sidebar_border_color,
+ user.profile_background_tile,
+ user.name,
+ user.profile_image_url,
+ user.created_at,
+ user.location,
+ user.follow_request_sent,
+ user.profile_link_color,
+ user.is_translator,
+ user.id_str,
+ user.default_profile,
+ user.contributors_enabled,
+ user.favourites_count,
+ user.url,
+ user.profile_image_url_https,
+ user.utc_offset,
+ user.id,
+ user.profile_use_background_image,
+ user.listed_count,
+ user.profile_text_color,
+ user.lang,
+ user.followers_count,
+ user.protected,
+ user.notifications,
+ user.profile_background_image_url_https,
+ user.profile_background_color,
+ user.verified,
+ user.geo_enabled,
+ user.time_zone,
+ user.description,
+ user.default_profile_image,
+ user.profile_background_image_url,
+ user.statuses_count,
+ user.friends_count,
+ user.following,
+ user.show_all_inline_media,
+ user.screen_name,
+ in_reply_to_screen_name,
+ source,
+ in_reply_to_status_id
+FROM
+ tweets;
\ No newline at end of file
[03/14] tajo git commit: initial work for TAJO-1591.
Posted by ji...@apache.org.
initial work for TAJO-1591.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/863e53ea
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/863e53ea
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/863e53ea
Branch: refs/heads/index_support
Commit: 863e53eae091bb6073efb31a1efd8acf4ff88ea0
Parents: 9b3824b
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu May 7 21:59:21 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu May 7 21:59:21 2015 -0700
----------------------------------------------------------------------
.../org/apache/tajo/catalog/CatalogUtil.java | 24 ++++----
.../org/apache/tajo/catalog/DDLBuilder.java | 6 +-
.../java/org/apache/tajo/catalog/TableDesc.java | 5 +-
.../java/org/apache/tajo/catalog/TableMeta.java | 8 +--
.../tajo/catalog/json/TableMetaAdapter.java | 7 +--
.../src/main/proto/CatalogProtos.proto | 2 +-
.../org/apache/tajo/catalog/TestTableDesc.java | 5 +-
.../org/apache/tajo/catalog/TestTableMeta.java | 12 ++--
.../tajo/catalog/store/HiveCatalogStore.java | 15 +++--
.../catalog/store/TestHiveCatalogStore.java | 22 +++----
.../dictionary/AbstractTableDescriptor.java | 2 +-
.../tajo/catalog/store/AbstractDBStore.java | 8 +--
.../org/apache/tajo/catalog/store/MemStore.java | 2 +-
.../org/apache/tajo/catalog/TestCatalog.java | 24 ++++----
.../org/apache/tajo/cli/tools/TajoDump.java | 2 +-
.../cli/tsql/commands/DescTableCommand.java | 3 +-
.../java/org/apache/tajo/benchmark/TPCH.java | 2 +-
.../tajo/engine/planner/global/DataChannel.java | 6 +-
.../engine/planner/global/GlobalPlanner.java | 10 ++--
.../planner/physical/ExternalSortExec.java | 4 +-
.../planner/physical/PhysicalPlanUtil.java | 54 ++++++++---------
.../physical/RangeShuffleFileWriteExec.java | 2 +-
.../org/apache/tajo/master/GlobalEngine.java | 2 +-
.../apache/tajo/master/exec/DDLExecutor.java | 3 +-
.../NonForwardQueryResultSystemScanner.java | 2 +-
.../apache/tajo/master/exec/QueryExecutor.java | 6 +-
.../java/org/apache/tajo/querymaster/Query.java | 4 +-
.../tajo/querymaster/QueryMasterTask.java | 2 +-
.../apache/tajo/querymaster/Repartitioner.java | 4 +-
.../java/org/apache/tajo/querymaster/Stage.java | 4 +-
.../src/main/proto/TajoWorkerProtocol.proto | 2 +-
.../org/apache/tajo/BackendTestingUtil.java | 2 +-
.../apache/tajo/LocalTajoTestingUtility.java | 2 +-
.../org/apache/tajo/TajoTestingCluster.java | 2 +-
.../apache/tajo/cli/tools/TestDDLBuilder.java | 2 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 4 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 2 +-
.../engine/planner/TestLogicalOptimizer.java | 6 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 10 ++--
.../tajo/engine/planner/TestPlannerUtil.java | 9 ++-
.../planner/global/TestBroadcastJoinPlan.java | 2 +-
.../planner/physical/TestBNLJoinExec.java | 4 +-
.../planner/physical/TestBSTIndexExec.java | 2 +-
.../planner/physical/TestExternalSortExec.java | 2 +-
.../physical/TestFullOuterHashJoinExec.java | 8 +--
.../physical/TestFullOuterMergeJoinExec.java | 11 ++--
.../planner/physical/TestHashAntiJoinExec.java | 4 +-
.../planner/physical/TestHashJoinExec.java | 4 +-
.../planner/physical/TestHashSemiJoinExec.java | 4 +-
.../physical/TestLeftOuterHashJoinExec.java | 8 +--
.../physical/TestLeftOuterNLJoinExec.java | 8 +--
.../planner/physical/TestMergeJoinExec.java | 4 +-
.../engine/planner/physical/TestNLJoinExec.java | 4 +-
.../planner/physical/TestPhysicalPlanner.java | 14 ++---
.../physical/TestProgressExternalSortExec.java | 2 +-
.../physical/TestRightOuterHashJoinExec.java | 6 +-
.../physical/TestRightOuterMergeJoinExec.java | 10 ++--
.../engine/planner/physical/TestSortExec.java | 2 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 6 +-
.../tajo/engine/query/TestHBaseTable.java | 10 ++--
.../org/apache/tajo/jdbc/TestResultSet.java | 3 +-
.../tajo/master/TestExecutionBlockCursor.java | 2 +-
.../apache/tajo/master/TestGlobalPlanner.java | 2 +-
.../org/apache/tajo/storage/TestRowFile.java | 2 +-
.../ws/rs/resources/TestTablesResource.java | 31 ++++------
.../org/apache/tajo/plan/LogicalPlanner.java | 8 +--
.../function/python/PythonScriptEngine.java | 2 +-
.../tajo/plan/logical/PersistentStoreNode.java | 8 +--
.../tajo/plan/logical/ShuffleFileWriteNode.java | 2 +-
.../tajo/plan/logical/StoreTableNode.java | 2 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 16 ++---
.../plan/verifier/PreLogicalPlanVerifier.java | 3 +-
tajo-plan/src/main/proto/Plan.proto | 2 +-
.../org/apache/tajo/plan/TestLogicalNode.java | 6 +-
.../org/apache/tajo/storage/StorageManager.java | 47 +++++----------
.../storage/hbase/AddSortForInsertRewriter.java | 3 +-
.../tajo/storage/hbase/HBasePutAppender.java | 3 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 3 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 3 +-
.../tajo/storage/hbase/TestColumnMapping.java | 3 +-
.../storage/hbase/TestHBaseStorageManager.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 7 +--
.../tajo/storage/TestCompressionStorages.java | 22 +++----
.../tajo/storage/TestDelimitedTextFile.java | 8 +--
.../tajo/storage/TestFileStorageManager.java | 10 ++--
.../apache/tajo/storage/TestFileSystems.java | 2 +-
.../org/apache/tajo/storage/TestLineReader.java | 8 +--
.../apache/tajo/storage/TestMergeScanner.java | 37 ++++++------
.../org/apache/tajo/storage/TestStorages.java | 62 ++++++++++----------
.../apache/tajo/storage/avro/TestAvroUtil.java | 7 +--
.../apache/tajo/storage/index/TestBSTIndex.java | 10 ++--
.../index/TestSingleCSVFileBSTIndex.java | 4 +-
.../apache/tajo/storage/json/TestJsonSerDe.java | 2 +-
93 files changed, 342 insertions(+), 398 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 12f36b1..5c1b9b3 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -295,13 +295,13 @@ public class CatalogUtil {
}
}
- public static TableMeta newTableMeta(StoreType type) {
- KeyValueSet defaultProperties = CatalogUtil.newPhysicalProperties(type);
- return new TableMeta(type, defaultProperties);
+ public static TableMeta newTableMeta(String storeType) {
+ KeyValueSet defaultProperties = CatalogUtil.newPhysicalProperties(storeType);
+ return new TableMeta(storeType, defaultProperties);
}
- public static TableMeta newTableMeta(StoreType type, KeyValueSet options) {
- return new TableMeta(type, options);
+ public static TableMeta newTableMeta(String storeType, KeyValueSet options) {
+ return new TableMeta(storeType, options);
}
public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path) {
@@ -851,21 +851,21 @@ public class CatalogUtil {
/**
* Create new table property with default configs. It is used to make TableMeta to be stored in Catalog.
*
- * @param type StoreType
+ * @param storeType StoreType
* @return Table properties
*/
- public static KeyValueSet newPhysicalProperties(StoreType type) {
+ public static KeyValueSet newPhysicalProperties(String storeType) {
KeyValueSet options = new KeyValueSet();
- if (StoreType.CSV == type || StoreType.TEXTFILE == type) {
+ if (storeType.equalsIgnoreCase("CSV") || storeType.equalsIgnoreCase("TEXT")) {
options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- } else if (StoreType.JSON == type) {
+ } else if (storeType.equalsIgnoreCase("JSON")) {
options.set(StorageConstants.TEXT_SERDE_CLASS, "org.apache.tajo.storage.json.JsonLineSerDe");
- } else if (StoreType.RCFILE == type) {
+ } else if (storeType.equalsIgnoreCase("RCFILE")) {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
- } else if (StoreType.SEQUENCEFILE == type) {
+ } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) {
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- } else if (type == StoreType.PARQUET) {
+ } else if (storeType.equalsIgnoreCase("PARQUET")) {
options.set(BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE);
options.set(PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE);
options.set(COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index 1a59e88..65640b9 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -31,7 +31,7 @@ public class DDLBuilder {
sb.append("--\n")
.append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
- .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType()));
+ .append(" Storage: ").append(desc.getMeta().getStoreType());
sb.append("\n-- Path: ").append(desc.getPath());
sb.append("\n--\n");
sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName()));
@@ -54,7 +54,7 @@ public class DDLBuilder {
sb.append("--\n")
.append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
- .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType()));
+ .append(" Storage: ").append(desc.getMeta().getStoreType());
sb.append("\n--\n");
sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName()));
buildSchema(sb, desc.getSchema());
@@ -88,7 +88,7 @@ public class DDLBuilder {
}
private static void buildUsingClause(StringBuilder sb, TableMeta meta) {
- sb.append(" USING " + CatalogUtil.getStoreTypeString(meta.getStoreType()));
+ sb.append(" USING " + meta.getStoreType());
}
private static void buildWithClause(StringBuilder sb, TableMeta meta) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index ee096a2..17f9146 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@ -27,7 +27,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.ProtoObject;
@@ -68,8 +67,8 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
this(tableName, schema, meta, path, true);
}
- public TableDesc(String tableName, Schema schema, StoreType type, KeyValueSet options, URI path) {
- this(tableName, schema, new TableMeta(type, options), path);
+ public TableDesc(String tableName, Schema schema, String storeType, KeyValueSet options, URI path) {
+ this(tableName, schema, new TableMeta(storeType, options), path);
}
public TableDesc(TableDescProto proto) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
index 2d95e6b..59cba44 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
@@ -41,16 +41,16 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj
private TableProto proto = TableProto.getDefaultInstance();
private boolean viaProto = false;
- @Expose protected StoreType storeType;
+ @Expose protected String storeType;
@Expose protected KeyValueSet options;
private TableMeta() {
builder = TableProto.newBuilder();
}
- public TableMeta(StoreType type, KeyValueSet options) {
+ public TableMeta(String storeType, KeyValueSet options) {
this();
- this.storeType = type;
+ this.storeType = storeType;
this.options = new KeyValueSet(options);
}
@@ -59,7 +59,7 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj
viaProto = true;
}
- public StoreType getStoreType() {
+ public String getStoreType() {
TableProtoOrBuilder p = viaProto ? proto : builder;
if (this.storeType != null) {
return storeType;
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
index 3f4304a..2d164f6 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
@@ -36,18 +36,17 @@ public class TableMetaAdapter implements GsonSerDerAdapter<TableMeta> {
Preconditions.checkNotNull(json);
JsonObject jsonObject = json.getAsJsonObject();
- CatalogProtos.StoreType type = CatalogProtos.StoreType.valueOf(
- CommonGsonHelper.getOrDie(jsonObject, "store").getAsString());
+ String storeType = CommonGsonHelper.getOrDie(jsonObject, "store").getAsString();
KeyValueSet keyValueSet = context.deserialize(CommonGsonHelper.getOrDie(jsonObject, "options"), KeyValueSet.class);
- return new TableMeta(type, keyValueSet);
+ return new TableMeta(storeType, keyValueSet);
}
@Override
public JsonElement serialize(TableMeta src, Type typeOfSrc,
JsonSerializationContext context) {
JsonObject jsonObj = new JsonObject();
- jsonObj.addProperty("store", src.getStoreType().name());
+ jsonObj.addProperty("store", src.getStoreType());
jsonObj.add("options", context.serialize(src.getOptions(), KeyValueSet.class));
return jsonObj;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index c467b4e..b213916 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -80,7 +80,7 @@ message FragmentProto {
}
message TableProto {
- required StoreType storeType = 1;
+ required string storeType = 1;
optional KeyValueSetProto params = 2;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
index feeaadd..9d84de6 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
@@ -21,7 +21,6 @@ package org.apache.tajo.catalog;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -45,7 +44,7 @@ public class TestTableDesc {
schema = new Schema();
schema.addColumn("name", Type.BLOB);
schema.addColumn("addr", Type.TEXT);
- info = CatalogUtil.newTableMeta(StoreType.CSV);
+ info = CatalogUtil.newTableMeta("CSV");
path = new Path(CommonTestingUtil.getTestDir(), "table1");
desc = new TableDesc("table1", schema, info, path.toUri());
stats = new TableStats();
@@ -71,7 +70,7 @@ public class TestTableDesc {
Schema schema = new Schema();
schema.addColumn("name", Type.BLOB);
schema.addColumn("addr", Type.TEXT);
- TableMeta info = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta info = CatalogUtil.newTableMeta("CSV");
testClone(info);
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
index 904b4bc..e1c0158 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
@@ -32,7 +32,7 @@ public class TestTableMeta {
@Before
public void setUp() {
- meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ meta = CatalogUtil.newTableMeta("CSV");
}
@Test
@@ -40,7 +40,7 @@ public class TestTableMeta {
Schema schema1 = new Schema();
schema1.addColumn("name", Type.BLOB);
schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta1 = CatalogUtil.newTableMeta("CSV");
TableMeta meta2 = new TableMeta(meta1.getProto());
assertEquals(meta1, meta2);
@@ -51,7 +51,7 @@ public class TestTableMeta {
Schema schema1 = new Schema();
schema1.addColumn("name", Type.BLOB);
schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta1 = CatalogUtil.newTableMeta("CSV");
TableMeta meta2 = (TableMeta) meta1.clone();
assertEquals(meta1.getStoreType(), meta2.getStoreType());
@@ -63,7 +63,7 @@ public class TestTableMeta {
Schema schema1 = new Schema();
schema1.addColumn("name", Type.BLOB);
schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta1 = CatalogUtil.newTableMeta("CSV");
TableMeta meta2 = (TableMeta) meta1.clone();
@@ -72,7 +72,7 @@ public class TestTableMeta {
@Test
public void testGetStorageType() {
- assertEquals(StoreType.CSV, meta.getStoreType());
+ assertEquals("CSV", meta.getStoreType());
}
@Test
@@ -80,7 +80,7 @@ public class TestTableMeta {
Schema schema2 = new Schema();
schema2.addColumn("name", Type.BLOB);
schema2.addColumn("addr", Type.TEXT);
- TableMeta meta2 = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta2 = CatalogUtil.newTableMeta("CSV");
assertTrue(meta.equals(meta2));
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index 5b1a996..835a158 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -110,7 +110,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
org.apache.hadoop.hive.ql.metadata.Table table = null;
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
Path path = null;
- CatalogProtos.StoreType storeType = null;
+ String storeType = null;
org.apache.tajo.catalog.Schema schema = null;
KeyValueSet options = null;
TableStats stats = null;
@@ -185,12 +185,12 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
// set file output format
String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT);
- storeType = CatalogUtil.getStoreType(HiveCatalogUtil.getStoreType(fileOutputformat));
+ storeType = HiveCatalogUtil.getStoreType(fileOutputformat);
- if (storeType.equals(CatalogProtos.StoreType.TEXTFILE)) {
+ if (storeType.equalsIgnoreCase("TEXT")) {
options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat));
- } else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) {
+ } else if (storeType.equals("RCFILE")) {
options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) {
@@ -198,7 +198,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
} else if (ColumnarSerDe.class.getName().equals(serde)) {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
- } else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) {
+ } else if (storeType.equals("SEQUENCEFILE") ) {
options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
@@ -543,13 +543,12 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
}
} else {
- if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.PARQUET)) {
+ if (tableDesc.getMeta().getStoreType().equals("PARQUET")) {
sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName());
sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName());
sd.getSerdeInfo().setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName());
} else {
- throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType
- ().name()));
+ throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType()));
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
index 05e4e0a..6e91b89 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
@@ -86,7 +86,7 @@ public class TestHiveCatalogStore {
@Test
public void testTableUsingTextFile() throws Exception {
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+ TableMeta meta = new TableMeta("CSV", new KeyValueSet());
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("c_custkey", TajoDataTypes.Type.INT4);
@@ -120,7 +120,7 @@ public class TestHiveCatalogStore {
public void testTableUsingRCFileWithBinarySerde() throws Exception {
KeyValueSet options = new KeyValueSet();
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
+ TableMeta meta = new TableMeta("RCFILE", options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
@@ -149,7 +149,7 @@ public class TestHiveCatalogStore {
public void testTableUsingRCFileWithTextSerde() throws Exception {
KeyValueSet options = new KeyValueSet();
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
+ TableMeta meta = new TableMeta("RCFILE", options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
@@ -178,7 +178,7 @@ public class TestHiveCatalogStore {
KeyValueSet options = new KeyValueSet();
options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava("\u0002"));
options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava("\u0003"));
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
+ TableMeta meta = new TableMeta("CSV", options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4);
@@ -221,7 +221,7 @@ public class TestHiveCatalogStore {
@Test
public void testAddTableByPartition() throws Exception {
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+ TableMeta meta = new TableMeta("CSV", new KeyValueSet());
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("n_name", TajoDataTypes.Type.TEXT);
@@ -329,7 +329,7 @@ public class TestHiveCatalogStore {
@Test
public void testGetAllTableNames() throws Exception{
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+ TableMeta meta = new TableMeta("CSV", new KeyValueSet());
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("n_name", TajoDataTypes.Type.TEXT);
schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4);
@@ -357,7 +357,7 @@ public class TestHiveCatalogStore {
@Test
public void testDeleteTable() throws Exception {
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+ TableMeta meta = new TableMeta("CSV", new KeyValueSet());
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("n_name", TajoDataTypes.Type.TEXT);
schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4);
@@ -381,7 +381,7 @@ public class TestHiveCatalogStore {
public void testTableUsingSequenceFileWithBinarySerde() throws Exception {
KeyValueSet options = new KeyValueSet();
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
+ TableMeta meta = new TableMeta("SEQUENCEFILE", options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
@@ -410,7 +410,7 @@ public class TestHiveCatalogStore {
public void testTableUsingSequenceFileWithTextSerde() throws Exception {
KeyValueSet options = new KeyValueSet();
options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
+ TableMeta meta = new TableMeta("SEQUENCEFILE", options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4);
@@ -437,7 +437,7 @@ public class TestHiveCatalogStore {
@Test
public void testTableUsingParquet() throws Exception {
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.PARQUET, new KeyValueSet());
+ TableMeta meta = new TableMeta("PARQUET", new KeyValueSet());
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("c_custkey", TajoDataTypes.Type.INT4);
@@ -469,7 +469,7 @@ public class TestHiveCatalogStore {
public void testDataTypeCompatibility() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testDataTypeCompatibility");
- TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet());
+ TableMeta meta = new TableMeta("CSV", new KeyValueSet());
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("col1", TajoDataTypes.Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
index a8b384c..f2dc3cf 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
@@ -60,7 +60,7 @@ abstract class AbstractTableDescriptor implements TableDescriptor {
protected TableProto getTableProto() {
TableProto.Builder metaBuilder = TableProto.newBuilder();
- metaBuilder.setStoreType(StoreType.SYSTEM);
+ metaBuilder.setStoreType("SYSTEM");
metaBuilder.setParams(KeyValueSetProto.newBuilder().build());
return metaBuilder.build();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index b10e5a5..7fe6ef3 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -764,7 +764,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt.setString(2, tableName);
pstmt.setString(3, TableType.EXTERNAL_TABLE.name());
pstmt.setString(4, table.getPath());
- pstmt.setString(5, CatalogUtil.getStoreTypeString(table.getMeta().getStoreType()));
+ pstmt.setString(5, table.getMeta().getStoreType());
pstmt.executeUpdate();
pstmt.close();
} else {
@@ -778,7 +778,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
pstmt.setInt(1, dbid);
pstmt.setString(2, tableName);
pstmt.setString(3, TableType.BASE_TABLE.name());
- pstmt.setString(4, CatalogUtil.getStoreTypeString(table.getMeta().getStoreType()));
+ pstmt.setString(4, table.getMeta().getStoreType());
pstmt.executeUpdate();
pstmt.close();
}
@@ -1567,7 +1567,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
PreparedStatement pstmt = null;
CatalogProtos.TableDescProto.Builder tableBuilder = null;
- StoreType storeType;
+ String storeType;
try {
tableBuilder = CatalogProtos.TableDescProto.newBuilder();
@@ -1609,7 +1609,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
tableBuilder.setPath(res.getString(4).trim());
}
- storeType = CatalogUtil.getStoreType(res.getString(5).trim());
+ storeType = res.getString(5).trim();
res.close();
pstmt.close();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
index 821b00c..55c4336 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
@@ -429,7 +429,7 @@ public class MemStore implements CatalogStore {
builder.setName(tableName);
builder.setPath(tableDesc.getPath());
builder.setTableType(tableDesc.getIsExternal()?"EXTERNAL":"BASE");
- builder.setStoreType(CatalogUtil.getStoreTypeString(tableDesc.getMeta().getStoreType()));
+ builder.setStoreType(tableDesc.getMeta().getStoreType());
tableList.add(builder.build());
tableId++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 306f581..cbcec83 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -264,7 +264,7 @@ public class TestCatalog {
TableDesc table = new TableDesc(
CatalogUtil.buildFQName(databaseName, tableName),
schema1,
- new TableMeta(StoreType.CSV, new KeyValueSet()),
+ new TableMeta("CSV", new KeyValueSet()),
path.toUri(), true);
return table;
}
@@ -317,7 +317,7 @@ public class TestCatalog {
TableDesc table = new TableDesc(
CatalogUtil.buildFQName(databaseName, tableName),
schema,
- new TableMeta(StoreType.CSV, new KeyValueSet()),
+ new TableMeta("CSV", new KeyValueSet()),
path.toUri(), true);
assertTrue(catalog.createTable(table));
@@ -331,7 +331,7 @@ public class TestCatalog {
table = new TableDesc(
CatalogUtil.buildFQName(databaseName, tableName),
schema,
- new TableMeta(StoreType.CSV, new KeyValueSet()),
+ new TableMeta("CSV", new KeyValueSet()),
path.toUri(), true);
assertTrue(catalog.createTable(table));
@@ -421,7 +421,7 @@ public class TestCatalog {
TableDesc meta = new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"),
schema1,
- StoreType.CSV,
+ "CSV",
new KeyValueSet(),
path.toUri());
@@ -441,7 +441,7 @@ public class TestCatalog {
TableDesc tableDesc = new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName),
schema,
- StoreType.CSV,
+ "CSV",
new KeyValueSet(),
path.toUri());
@@ -561,7 +561,7 @@ public class TestCatalog {
String tableName = "indexed";
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
return new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta,
new Path(CommonTestingUtil.getTestDir(), "indexed").toUri());
@@ -732,7 +732,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV", opts);
Schema partSchema = new Schema();
@@ -772,7 +772,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV", opts);
Schema partSchema = new Schema();
partSchema.addColumn("id", Type.INT4);
@@ -810,7 +810,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV", opts);
Schema partSchema = new Schema();
partSchema.addColumn("id", Type.INT4);
@@ -847,7 +847,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV", opts);
Schema partSchema = new Schema();
partSchema.addColumn("id", Type.INT4);
@@ -884,7 +884,7 @@ public class TestCatalog {
String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable");
KeyValueSet opts = new KeyValueSet();
opts.set("file.delimiter", ",");
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV", opts);
Schema partSchema = new Schema();
partSchema.addColumn("id", Type.INT4);
@@ -999,7 +999,7 @@ public class TestCatalog {
TableDesc setPropertyDesc = catalog.getTableDesc("default","mynewcooltable");
KeyValueSet options = new KeyValueSet();
options.set("timezone", "GMT+9"); // Seoul, Korea
- setPropertyDesc.setMeta(new TableMeta(StoreType.CSV, options));
+ setPropertyDesc.setMeta(new TableMeta("CSV", options));
String prevTimeZone = setPropertyDesc.getMeta().getOption("timezone");
String newTimeZone = "GMT-7"; // Silicon Valley, California
catalog.alterTable(createMockAlterTableSetProperty(newTimeZone));
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
index 497ccb6..7fefc5a 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java
@@ -177,7 +177,7 @@ public class TajoDump {
try {
TableDesc table = client.getTableDesc(CatalogUtil.buildFQName(databaseName, tableName));
- if (table.getMeta().getStoreType() == StoreType.SYSTEM) {
+ if (table.getMeta().getStoreType().equalsIgnoreCase("SYSTEM")) {
continue;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index a3960e6..f888c36 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@ -21,7 +21,6 @@ package org.apache.tajo.cli.tsql.commands;
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tajo.TajoConstants;
-import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
@@ -80,7 +79,7 @@ public class DescTableCommand extends TajoShellCommand {
StringBuilder sb = new StringBuilder();
sb.append("\ntable name: ").append(desc.getName()).append("\n");
sb.append("table path: ").append(desc.getPath()).append("\n");
- sb.append("store type: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())).append("\n");
+ sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n");
if (desc.getStats() != null) {
long row = desc.getStats().getNumRows();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java
index 609e49c..9994634 100644
--- a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java
+++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java
@@ -207,7 +207,7 @@ public class TPCH extends BenchmarkSet {
}
public void loadTable(String tableName) throws ServiceException {
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
meta.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
PartitionMethodDesc partitionMethodDesc = null;
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
index 11548d3..3738dd3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
@@ -40,7 +40,7 @@ public class DataChannel {
private Schema schema;
- private StoreType storeType = StoreType.RAW;
+ private String storeType = "RAW";
public DataChannel(ExecutionBlockId srcId, ExecutionBlockId targetId) {
this.srcId = srcId;
@@ -144,11 +144,11 @@ public class DataChannel {
return this.storeType != null;
}
- public void setStoreType(StoreType storeType) {
+ public void setStoreType(String storeType) {
this.storeType = storeType;
}
- public StoreType getStoreType() {
+ public String getStoreType() {
return storeType;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 36bdf21..e18b2eb 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -69,7 +69,7 @@ public class GlobalPlanner {
private static Log LOG = LogFactory.getLog(GlobalPlanner.class);
private final TajoConf conf;
- private final CatalogProtos.StoreType storeType;
+ private final String storeType;
private final CatalogService catalog;
private final GlobalPlanRewriteEngine rewriteEngine;
@@ -77,7 +77,7 @@ public class GlobalPlanner {
public GlobalPlanner(final TajoConf conf, final CatalogService catalog) throws IOException {
this.conf = conf;
this.catalog = catalog;
- this.storeType = CatalogProtos.StoreType.valueOf(conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase());
+ this.storeType = conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase();
Preconditions.checkArgument(storeType != null);
Class<? extends GlobalPlanRewriteRuleProvider> clazz =
@@ -99,7 +99,7 @@ public class GlobalPlanner {
return catalog;
}
- public CatalogProtos.StoreType getStoreType() {
+ public String getStoreType() {
return storeType;
}
@@ -175,7 +175,7 @@ public class GlobalPlanner {
private static void setFinalOutputChannel(DataChannel outputChannel, Schema outputSchema) {
outputChannel.setShuffleType(NONE_SHUFFLE);
outputChannel.setShuffleOutputNum(1);
- outputChannel.setStoreType(CatalogProtos.StoreType.CSV);
+ outputChannel.setStoreType("CSV");
outputChannel.setSchema(outputSchema);
}
@@ -1184,7 +1184,7 @@ public class GlobalPlanner {
for (DataChannel dataChannel : masterPlan.getIncomingChannels(execBlock.getId())) {
// This data channel will be stored in staging directory, but RawFile, default file type, does not support
// distributed file system. It needs to change the file format for distributed file system.
- dataChannel.setStoreType(CatalogProtos.StoreType.CSV);
+ dataChannel.setStoreType("CSV");
ExecutionBlock subBlock = masterPlan.getExecBlock(dataChannel.getSrcId());
ProjectionNode copy = PlannerUtil.clone(plan, node);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
index 3da296c..13b5a3a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
@@ -110,7 +110,7 @@ public class ExternalSortExec extends SortExec {
super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys());
this.plan = plan;
- this.meta = CatalogUtil.newTableMeta(StoreType.ROWFILE);
+ this.meta = CatalogUtil.newTableMeta("ROWFILE");
this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT);
if (defaultFanout < 2) {
@@ -163,7 +163,7 @@ public class ExternalSortExec extends SortExec {
*/
private Path sortAndStoreChunk(int chunkId, List<Tuple> tupleBlock)
throws IOException {
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.RAW);
+ TableMeta meta = CatalogUtil.newTableMeta("RAW");
int rowNum = tupleBlock.size();
long sortStart = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
index 247b373..f2bd75f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
@@ -29,12 +29,15 @@ import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.PhysicalPlanningException;
-import org.apache.tajo.plan.util.PlannerUtil;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.NodeType;
import org.apache.tajo.plan.logical.PersistentStoreNode;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.storage.*;
+import org.apache.tajo.plan.util.PlannerUtil;
+import org.apache.tajo.storage.BaseTupleComparator;
+import org.apache.tajo.storage.FileStorageManager;
+import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.storage.TupleComparator;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
@@ -205,20 +208,15 @@ public class PhysicalPlanUtil {
* @param nullChar A character for NULL representation
*/
private static void setNullCharForTextSerializer(TableMeta meta, String nullChar) {
- switch (meta.getStoreType()) {
- case CSV:
- meta.putOption(StorageConstants.TEXT_NULL, nullChar);
- break;
- case TEXTFILE:
- meta.putOption(StorageConstants.TEXT_NULL, nullChar);
- break;
- case RCFILE:
- meta.putOption(StorageConstants.RCFILE_NULL, nullChar);
- break;
- case SEQUENCEFILE:
- meta.putOption(StorageConstants.SEQUENCEFILE_NULL, nullChar);
- break;
- default: // nothing to do
+ String storeType = meta.getStoreType();
+ if (storeType.equalsIgnoreCase("CSV")) {
+ meta.putOption(StorageConstants.TEXT_NULL, nullChar);
+ } else if (storeType.equalsIgnoreCase("TEXT")) {
+ meta.putOption(StorageConstants.TEXT_NULL, nullChar);
+ } else if (storeType.equalsIgnoreCase("RCFILE")) {
+ meta.putOption(StorageConstants.RCFILE_NULL, nullChar);
+ } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) {
+ meta.putOption(StorageConstants.SEQUENCEFILE_NULL, nullChar);
}
}
@@ -229,17 +227,17 @@ public class PhysicalPlanUtil {
* @return True if TableMeta contains NULL char property according to file format
*/
public static boolean containsNullChar(TableMeta meta) {
- switch (meta.getStoreType()) {
- case CSV:
- return meta.containsOption(StorageConstants.TEXT_NULL);
- case TEXTFILE:
- return meta.containsOption(StorageConstants.TEXT_NULL);
- case RCFILE:
- return meta.containsOption(StorageConstants.RCFILE_NULL);
- case SEQUENCEFILE:
- return meta.containsOption(StorageConstants.SEQUENCEFILE_NULL);
- default: // nothing to do
- return false;
+ String storeType = meta.getStoreType();
+ if (storeType.equalsIgnoreCase("CSV")) {
+ return meta.containsOption(StorageConstants.TEXT_NULL);
+ } else if (storeType.equalsIgnoreCase("TEXT")) {
+ return meta.containsOption(StorageConstants.TEXT_NULL);
+ } else if (storeType.equalsIgnoreCase("RCFILE")) {
+ return meta.containsOption(StorageConstants.RCFILE_NULL);
+ } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) {
+ return meta.containsOption(StorageConstants.SEQUENCEFILE_NULL);
+ } else {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
index 8da1a03..cb1fa05 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
@@ -75,7 +75,7 @@ public class RangeShuffleFileWriteExec extends UnaryPhysicalExec {
Path storeTablePath = new Path(context.getWorkDir(), "output");
LOG.info("Output data directory: " + storeTablePath);
this.meta = CatalogUtil.newTableMeta(context.getDataChannel() != null ?
- context.getDataChannel().getStoreType() : CatalogProtos.StoreType.RAW);
+ context.getDataChannel().getStoreType() : "RAW");
FileSystem fs = new RawLocalFileSystem();
fs.mkdirs(storeTablePath);
this.appender = (FileAppender) ((FileStorageManager)StorageManager.getFileStorageManager(context.getConf()))
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 074f34e..adbc8a4 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -293,7 +293,7 @@ public class GlobalEngine extends AbstractService {
}
private void verifyInsertTableSchema(QueryContext queryContext, VerificationState state, LogicalPlan plan) {
- StoreType storeType = PlannerUtil.getStoreType(plan);
+ String storeType = PlannerUtil.getStoreType(plan);
if (storeType != null) {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
if (rootNode.getChild().getType() == NodeType.INSERT) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
index 9264c68..c84b0c7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
@@ -29,7 +29,6 @@ import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.exception.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
-import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
@@ -210,7 +209,7 @@ public class DDLExecutor {
createTable.getPartitionMethod(), ifNotExists);
}
- public TableDesc createTable(QueryContext queryContext, String tableName, CatalogProtos.StoreType storeType,
+ public TableDesc createTable(QueryContext queryContext, String tableName, String storeType,
Schema schema, TableMeta meta, Path path, boolean isExternal,
PartitionMethodDesc partitionDesc, boolean ifNotExists) throws IOException {
String databaseName;
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
index 958c252..307a155 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
@@ -120,7 +120,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
.createPlan(taskContext, leafBlock.getPlan());
tableDesc = new TableDesc("table_"+System.currentTimeMillis(), physicalExec.getSchema(),
- new TableMeta(StoreType.SYSTEM, new KeyValueSet()), null);
+ new TableMeta("SYSTEM", new KeyValueSet()), null);
outSchema = physicalExec.getSchema();
encoder = RowStoreUtil.createEncoder(getLogicalSchema());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
index ad1a8e3..481bdbe 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
@@ -415,7 +415,7 @@ public class QueryExecutor {
List<CatalogProtos.ColumnProto> columns = new ArrayList<CatalogProtos.ColumnProto>();
CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder()
.setTableName(nodeUniqName)
- .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType(CatalogProtos.StoreType.CSV).build())
+ .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType("CSV").build())
.setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build())
.setStats(stats.getProto())
.build();
@@ -436,7 +436,7 @@ public class QueryExecutor {
SubmitQueryResponse.Builder responseBuilder) throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
- CatalogProtos.StoreType storeType = PlannerUtil.getStoreType(plan);
+ String storeType = PlannerUtil.getStoreType(plan);
if (storeType != null) {
StorageManager sm = StorageManager.getStorageManager(context.getConf(), storeType);
StorageProperty storageProperty = sm.getStorageProperty();
@@ -474,7 +474,7 @@ public class QueryExecutor {
public static MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner)
throws Exception {
- CatalogProtos.StoreType storeType = PlannerUtil.getStoreType(plan);
+ String storeType = PlannerUtil.getStoreType(plan);
if (storeType != null) {
StorageManager sm = StorageManager.getStorageManager(planner.getConf(), storeType);
StorageProperty storageProperty = sm.getStorageProperty();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
index 1ce15fc..a2e434b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
@@ -420,7 +420,7 @@ public class Query implements EventHandler<QueryEvent> {
if (finalState != QueryState.QUERY_SUCCEEDED) {
Stage lastStage = query.getStage(stageEvent.getExecutionBlockId());
if (lastStage != null && lastStage.getTableMeta() != null) {
- StoreType storeType = lastStage.getTableMeta().getStoreType();
+ String storeType = lastStage.getTableMeta().getStoreType();
if (storeType != null) {
LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot();
try {
@@ -439,7 +439,7 @@ public class Query implements EventHandler<QueryEvent> {
private QueryState finalizeQuery(Query query, QueryCompletedEvent event) {
Stage lastStage = query.getStage(event.getExecutionBlockId());
- StoreType storeType = lastStage.getTableMeta().getStoreType();
+ String storeType = lastStage.getTableMeta().getStoreType();
try {
LogicalRootNode rootNode = lastStage.getMasterPlan().getLogicalPlan().getRootBlock().getRoot();
CatalogService catalog = lastStage.getContext().getQueryMasterContext().getWorkerContext().getCatalog();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
index 465fa84..8d20141 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
@@ -322,7 +322,7 @@ public class QueryMasterTask extends CompositeService {
jsonExpr = null; // remove the possible OOM
plan = planner.createPlan(queryContext, expr);
- StoreType storeType = PlannerUtil.getStoreType(plan);
+ String storeType = PlannerUtil.getStoreType(plan);
if (storeType != null) {
sm = StorageManager.getStorageManager(systemConf, storeType);
StorageProperty storageProperty = sm.getStorageProperty();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
index 8e9e343..986d16a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
@@ -232,7 +232,7 @@ public class Repartitioner {
int maxStatsScanIdx = -1;
StringBuilder nonLeafScanNamesBuilder = new StringBuilder();
for (int i = 0; i < scans.length; i++) {
- if (scans[i].getTableDesc().getMeta().getStoreType() == StoreType.RAW) {
+ if (scans[i].getTableDesc().getMeta().getStoreType().equalsIgnoreCase("RAW")) {
// Intermediate data scan
hasNonLeafNode = true;
largeScanIndexList.add(i);
@@ -670,7 +670,7 @@ public class Repartitioner {
TupleRange mergedRange = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats(), false);
if (sortNode.getSortPurpose() == SortPurpose.STORAGE_SPECIFIED) {
- StoreType storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan());
+ String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan());
CatalogService catalog = stage.getContext().getQueryMasterContext().getWorkerContext().getCatalog();
LogicalRootNode rootNode = masterPlan.getLogicalPlan().getRootBlock().getRoot();
TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, rootNode.getChild());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index d82d078..f03ee2f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -721,10 +721,10 @@ public class Stage implements EventHandler<StageEvent> {
DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0);
// if store plan (i.e., CREATE or INSERT OVERWRITE)
- StoreType storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan());
+ String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan());
if (storeType == null) {
// get default or store type
- storeType = StoreType.CSV;
+ storeType = "CSV";
}
schema = channel.getSchema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
index fddef8f..e660a30 100644
--- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto
+++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
@@ -189,7 +189,7 @@ message DataChannelProto {
repeated ColumnProto shuffleKeys = 7;
optional int32 numOutputs = 9 [default = 1];
- optional StoreType storeType = 10 [default = CSV];
+ optional string storeType = 10;
}
message RunExecutionBlockRequestProto {
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
index 308ed28..0774eff 100644
--- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
@@ -42,7 +42,7 @@ public class BackendTestingUtil {
mockupSchema = new Schema();
mockupSchema.addColumn("deptname", Type.TEXT);
mockupSchema.addColumn("score", Type.INT4);
- mockupMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ mockupMeta = CatalogUtil.newTableMeta("CSV");
}
public static void writeTmpTable(TajoConf conf, Path tablePath)
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
index 5407ff5..6691df1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
+++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -116,7 +116,7 @@ public class LocalTajoTestingUtility {
fs.mkdirs(tablePath);
Path dfsPath = new Path(tablePath, localPath.getName());
fs.copyFromLocalFile(localPath, dfsPath);
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.TEXTFILE, option);
+ TableMeta meta = CatalogUtil.newTableMeta("TEXT", option);
// Add fake table statistic data to tables.
// It gives more various situations to unit tests.
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 54f9e92..59f6da5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -669,7 +669,7 @@ public class TajoTestingCluster {
out.close();
}
}
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV, tableOption);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV", tableOption);
client.createExternalTable(tableName, schema, tablePath.toUri(), meta);
} finally {
client.close();
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
index 704980b..aac5f90 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
@@ -41,7 +41,7 @@ public class TestDDLBuilder {
schema1.addColumn("name", TajoDataTypes.Type.BLOB);
schema1.addColumn("addr", TajoDataTypes.Type.TEXT);
- meta1 = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
+ meta1 = CatalogUtil.newTableMeta("CSV");
meta1.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 36ffd0c..af86abc 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -264,8 +264,8 @@ public class ExprTestBase {
vtuple.put(i, lazyTuple.get(i));
}
}
- cat.createTable(new TableDesc(qualifiedTableName, inputSchema,
- CatalogProtos.StoreType.CSV, new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()));
+ cat.createTable(new TableDesc(qualifiedTableName, inputSchema,"CSV",
+ new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()));
}
Target [] targets;
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
index 0466a24..dd5a75a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
@@ -103,7 +103,7 @@ public class TestEvalTreeUtil {
schema.addColumn("score", TajoDataTypes.Type.INT4);
schema.addColumn("age", TajoDataTypes.Type.INT4);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
TableDesc desc = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
index c9b52fd..a408fd6 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
@@ -79,7 +79,7 @@ public class TestLogicalOptimizer {
schema3.addColumn("score", Type.INT4);
schema3.addColumn("phone", Type.INT4);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
@@ -87,13 +87,13 @@ public class TestLogicalOptimizer {
TableDesc student =
new TableDesc(
- CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(),
+ CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", new KeyValueSet(),
CommonTestingUtil.getTestDir().toUri());
catalog.createTable(student);
TableDesc score =
new TableDesc(
- CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(),
+ CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", new KeyValueSet(),
CommonTestingUtil.getTestDir().toUri());
catalog.createTable(score);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index af0aa6a..d5d5779 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -92,19 +92,19 @@ public class TestLogicalPlanner {
schema3.addColumn("deptname", Type.TEXT);
schema3.addColumn("score", Type.INT4);
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
catalog.createTable(people);
TableDesc student = new TableDesc(
- CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(),
+ CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV", new KeyValueSet(),
CommonTestingUtil.getTestDir().toUri());
catalog.createTable(student);
TableDesc score = new TableDesc(
- CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(),
+ CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV", new KeyValueSet(),
CommonTestingUtil.getTestDir().toUri());
catalog.createTable(score);
@@ -121,7 +121,7 @@ public class TestLogicalPlanner {
tpch.loadSchemas();
tpch.loadOutSchema();
for (String table : tpchTables) {
- TableMeta m = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta m = CatalogUtil.newTableMeta("CSV");
TableDesc d = CatalogUtil.newTableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m,
CommonTestingUtil.getTestDir());
@@ -979,7 +979,7 @@ public class TestLogicalPlanner {
assertEquals(Type.INT8, def.getColumn(2).getDataType().getType());
assertEquals("score", def.getColumn(3).getSimpleName());
assertEquals(Type.FLOAT4, def.getColumn(3).getDataType().getType());
- assertEquals(StoreType.CSV, createTable.getStorageType());
+ assertTrue("CSV".equalsIgnoreCase(createTable.getStorageType()));
assertEquals("/tmp/data", createTable.getPath().toString());
assertTrue(createTable.hasOptions());
assertEquals("|", createTable.getOptions().get("csv.delimiter"));
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
index 9ce7b5b..0082800 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
@@ -29,7 +29,6 @@ import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.function.builtin.SumInt;
@@ -40,10 +39,10 @@ import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleComparator;
import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
@@ -85,7 +84,7 @@ public class TestPlannerUtil {
schema3.addColumn("deptname", Type.TEXT);
schema3.addColumn("score", CatalogUtil.newSimpleDataType(Type.INT4));
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta("CSV");
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
CommonTestingUtil.getTestDir().toUri());
@@ -93,13 +92,13 @@ public class TestPlannerUtil {
TableDesc student =
new TableDesc(
- CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV,
+ CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "CSV",
new KeyValueSet(), CommonTestingUtil.getTestDir().toUri());
catalog.createTable(student);
TableDesc score =
new TableDesc(
- CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV,
+ CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "CSV",
new KeyValueSet(), CommonTestingUtil.getTestDir().toUri());
catalog.createTable(score);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
index eb92bcf..9ff360f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
@@ -128,7 +128,7 @@ public class TestBroadcastJoinPlan {
}
private TableDesc makeTestData(String tableName, Schema schema, int dataSize) throws Exception {
- TableMeta tableMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
+ TableMeta tableMeta = CatalogUtil.newTableMeta("CSV");
Path dataPath = new Path(testDir, tableName + ".csv");
String contentsData = "";
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
index 842a120..a50d813 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
@@ -85,7 +85,7 @@ public class TestBNLJoinExec {
schema.addColumn("memid", Type.INT4);
schema.addColumn("deptname", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, schema, employeePath);
@@ -107,7 +107,7 @@ public class TestBNLJoinExec {
peopleSchema.addColumn("fk_memid", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta("CSV");
Path peoplePath = new Path(testDir, "people.csv");
appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(peopleMeta, peopleSchema, peoplePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
index b22a87e..ac8860f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
@@ -112,7 +112,7 @@ public class TestBSTIndexExec {
writer.open();
long offset;
- meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ meta = CatalogUtil.newTableMeta("CSV");
tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv");
fs = tablePath.getFileSystem(conf);
fs.mkdirs(tablePath.getParent());
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index 0ef8294..0b93c76 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -81,7 +81,7 @@ public class TestExternalSortExec {
schema.addColumn("empid", Type.INT4);
schema.addColumn("deptname", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta("CSV");
Path employeePath = new Path(testDir, "employee.csv");
Appender appender = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(employeeMeta, schema, employeePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/863e53ea/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index 780e698..e5ba9b3 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -103,7 +103,7 @@ public class TestFullOuterHashJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(dep3Meta, dep3Schema, dep3Path);
@@ -133,7 +133,7 @@ public class TestFullOuterHashJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(job3Meta, job3Schema, job3Path);
@@ -173,7 +173,7 @@ public class TestFullOuterHashJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(emp3Meta, emp3Schema, emp3Path);
@@ -226,7 +226,7 @@ public class TestFullOuterHashJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV");
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 = ((FileStorageManager)StorageManager.getFileStorageManager(conf))
.getAppender(phone3Meta, phone3Schema, phone3Path);
[09/14] tajo git commit: Add error log message to exception.
Posted by ji...@apache.org.
Add error log message to exception.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/70df3a8d
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/70df3a8d
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/70df3a8d
Branch: refs/heads/index_support
Commit: 70df3a8d2a5d6b6cef5ae233a546d8a405780964
Parents: d9776b3
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sun May 10 21:38:02 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sun May 10 21:38:02 2015 -0700
----------------------------------------------------------------------
.../tajo/plan/function/python/PythonScriptEngine.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/70df3a8d/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
index 49dce87..8980de1 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
@@ -478,7 +478,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
@@ -512,7 +512,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -532,7 +532,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -552,7 +552,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext(functionContext);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -573,7 +573,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
return outputHandler.getPartialResultString();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -595,7 +595,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
[08/14] tajo git commit: TAJO-1593: Add missing stop condition to
Taskrunner. (jinho)
Posted by ji...@apache.org.
TAJO-1593: Add missing stop condition to Taskrunner. (jinho)
Closes #562
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ddd39213
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ddd39213
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ddd39213
Branch: refs/heads/index_support
Commit: ddd39213d87fd76e9e80099e92010fd71b3d363e
Parents: 1baf8dc
Author: Jinho Kim <jh...@apache.org>
Authored: Fri May 8 23:42:03 2015 +0900
Committer: Jinho Kim <jh...@apache.org>
Committed: Fri May 8 23:42:03 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../apache/tajo/worker/ExecutionBlockContext.java | 15 ++++++++++-----
.../main/java/org/apache/tajo/worker/TaskRunner.java | 1 +
.../java/org/apache/tajo/rpc/AsyncRpcServer.java | 6 +++++-
.../java/org/apache/tajo/rpc/BlockingRpcServer.java | 6 +++++-
5 files changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddd39213/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 9788307..0aad306 100644
--- a/CHANGES
+++ b/CHANGES
@@ -121,6 +121,8 @@ Release 0.11.0 - unreleased
BUG FIXES
+ TAJO-1593: Add missing stop condition to Taskrunner. (jinho)
+
TAJO-1556: "insert into select" with reordered column list does not work.
(Contributed by Yongjin Choi, Committed by jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddd39213/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
index 270000a..0cc3304 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
@@ -35,9 +35,8 @@ import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.QueryMasterProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.plan.serder.PlanProto;
-import org.apache.tajo.rpc.NettyClientBase;
-import org.apache.tajo.rpc.NullCallback;
-import org.apache.tajo.rpc.RpcClientManager;
+import org.apache.tajo.rpc.*;
+import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.storage.HashShuffleAppenderManager;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.NetUtils;
@@ -50,6 +49,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -307,7 +307,10 @@ public class ExecutionBlockContext {
getWorkerContext().getHashShuffleAppenderManager().close(ebId);
if (shuffles == null) {
reporterBuilder.addAllIntermediateEntries(intermediateEntries);
- stub.doneExecutionBlock(null, reporterBuilder.build(), NullCallback.get());
+
+ CallFuture<PrimitiveProtos.NullProto> callFuture = new CallFuture<PrimitiveProtos.NullProto>();
+ stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture);
+ callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
return;
}
@@ -340,7 +343,9 @@ public class ExecutionBlockContext {
}
}
try {
- stub.doneExecutionBlock(null, reporterBuilder.build(), NullCallback.get());
+ CallFuture<PrimitiveProtos.NullProto> callFuture = new CallFuture<PrimitiveProtos.NullProto>();
+ stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture);
+ callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (Throwable e) {
// can't send report to query master
LOG.fatal(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddd39213/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
index 31f25f0..774f358 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
@@ -241,6 +241,7 @@ public class TaskRunner extends AbstractService {
// immediately.
if (taskRequest.getShouldDie()) {
LOG.info("Received ShouldDie flag:" + getId());
+ break;
} else {
getContext().getWorkerContext().getWorkerSystemMetrics().counter("query", "task").inc();
LOG.info("Accumulated Received Task: " + (++receivedNum));
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddd39213/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
index 22f47b0..134b3cf 100644
--- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
+++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
@@ -142,7 +142,11 @@ public class AsyncRpcServer extends NettyServerBase {
ctx.close();
}
Throwable rootCause = ExceptionUtils.getRootCause(cause);
- LOG.fatal(ExceptionUtils.getMessage(rootCause), rootCause);
+ if(rootCause == null) {
+ LOG.fatal(ExceptionUtils.getMessage(cause), cause);
+ } else {
+ LOG.fatal(ExceptionUtils.getMessage(rootCause), rootCause);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/ddd39213/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcServer.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcServer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcServer.java
index 93c28e3..007ada5 100644
--- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcServer.java
+++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcServer.java
@@ -133,7 +133,11 @@ public class BlockingRpcServer extends NettyServerBase {
ctx.close();
}
Throwable rootCause = ExceptionUtils.getRootCause(cause);
- LOG.fatal(ExceptionUtils.getMessage(rootCause), rootCause);
+ if(rootCause == null) {
+ LOG.fatal(ExceptionUtils.getMessage(cause), cause);
+ } else {
+ LOG.fatal(ExceptionUtils.getMessage(rootCause), rootCause);
+ }
}
}
}
[07/14] tajo git commit: Merge branch 'master' of
github.com:apache/tajo into TAJO-1591
Posted by ji...@apache.org.
Merge branch 'master' of github.com:apache/tajo into TAJO-1591
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/d9776b32
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/d9776b32
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/d9776b32
Branch: refs/heads/index_support
Commit: d9776b32df0ef79b99087310ece41277ae517e9a
Parents: 285b179 1baf8dc
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri May 8 00:56:05 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri May 8 00:56:05 2015 -0700
----------------------------------------------------------------------
CHANGES | 10 +++
.../apache/tajo/function/FunctionSignature.java | 10 ++-
.../org/apache/tajo/function/FunctionUtil.java | 39 +++++++++
.../cli/tsql/DefaultTajoCliOutputFormatter.java | 4 +-
.../cli/tsql/commands/DescFunctionCommand.java | 18 +---
.../java/org/apache/tajo/util/ClassSize.java | 18 +++-
.../java/org/apache/tajo/util/NumberUtil.java | 54 ++++++++++++
.../java/org/apache/tajo/util/UnsafeUtil.java | 28 +++++++
.../apache/tajo/querymaster/Repartitioner.java | 18 ++--
.../java/org/apache/tajo/querymaster/Task.java | 56 ++++++-------
.../main/java/org/apache/tajo/util/JSPUtil.java | 14 ----
.../tajo/worker/ExecutionBlockContext.java | 32 ++-----
.../src/main/proto/TajoWorkerProtocol.proto | 16 +---
.../main/resources/webapps/admin/functions.jsp | 3 +-
.../cli/tsql/TestDefaultCliOutputFormatter.java | 13 ++-
.../apache/tajo/master/TestRepartitioner.java | 77 +++++++----------
.../tajo/querymaster/TestIntermediateEntry.java | 24 +++---
.../tajo/storage/HashShuffleAppender.java | 87 ++++++++++++++------
.../storage/HashShuffleAppenderManager.java | 12 +--
19 files changed, 316 insertions(+), 217 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/d9776b32/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/d9776b32/tajo-core/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
[05/14] tajo git commit: Fixed some literal string to constants.
Posted by ji...@apache.org.
Fixed some literal string to constants.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/285b1790
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/285b1790
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/285b1790
Branch: refs/heads/index_support
Commit: 285b179067fcd52fa1eead1a4f4b04da3813182b
Parents: 863e53e
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri May 8 00:50:17 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri May 8 00:50:17 2015 -0700
----------------------------------------------------------------------
.../tajo/catalog/store/HiveCatalogStore.java | 9 +++---
.../java/org/apache/tajo/BuiltinStorages.java | 32 ++++++++++++++++++++
2 files changed, 37 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/285b1790/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index 835a158..51d1044 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.exception.*;
@@ -472,7 +473,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
table.setPartitionKeys(partitionKeys);
}
- if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) {
+ if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.RCFILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
@@ -487,7 +488,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT,
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
- } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV)
+ } else if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.CSV)
|| tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.TEXTFILE)) {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
@@ -512,7 +513,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL)));
table.getParameters().remove(StorageConstants.TEXT_NULL);
}
- } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE)) {
+ } else if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) {
String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName());
@@ -543,7 +544,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL);
}
} else {
- if (tableDesc.getMeta().getStoreType().equals("PARQUET")) {
+ if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.PARQUET)) {
sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName());
sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName());
sd.getSerdeInfo().setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName());
http://git-wip-us.apache.org/repos/asf/tajo/blob/285b1790/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java
new file mode 100644
index 0000000..318270c
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/BuiltinStorages.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+public class BuiltinStorages {
+ public static final String CSV = "CSV";
+ public static final String TEXT = "TEXT";
+ public static final String JSON = "JSON";
+ public static final String RAW = "RAW";
+ public static final String RCFILE = "RCFILE";
+ public static final String ROW = "ROW";
+ public static final String PARQUET = "PARQUET";
+ public static final String SEQUENCE_FILE = "SEQUENCEFILE";
+ public static final String AVRO = "AVRO";
+ public static final String HBASE = "HBASE";
+}
[10/14] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
index 204f607..0d1c94a 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
@@ -23,6 +23,7 @@ import io.netty.buffer.ByteBuf;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;
+import org.apache.tajo.catalog.*;
import org.apache.commons.net.util.Base64;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
@@ -35,26 +36,212 @@ import org.apache.tajo.datum.TextDatum;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.text.TextLineDeserializer;
import org.apache.tajo.storage.text.TextLineParsingError;
+import org.apache.tajo.util.StringUtils;
+import org.apache.tajo.util.TUtil;
import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
public class JsonLineDeserializer extends TextLineDeserializer {
private JSONParser parser;
- private Type[] types;
- private String[] columnNames;
+ // Full Path -> Type
+ private Map<String, Type> types;
+ private String [] projectedPaths;
- public JsonLineDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- super(schema, meta, targetColumnIndexes);
+ public JsonLineDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ super(schema, meta);
+
+ projectedPaths = new String[projected.length];
+ for (int i = 0; i < projected.length; i++) {
+ this.projectedPaths[i] = projected[i].getSimpleName();
+ }
}
@Override
public void init() {
- types = SchemaUtil.toTypes(schema);
- columnNames = SchemaUtil.toSimpleNames(schema);
+ types = TUtil.newHashMap();
+ for (Column column : schema.getAllColumns()) {
+ // Keep types which only belong to projected paths
+ // For example, assume that a projected path is 'name/first_name', where name is RECORD and first_name is TEXT.
+ // In this case, we should keep two types:
+ // * name - RECORD
+ // * name/first_name TEXT
+ for (String p :projectedPaths) {
+ if (p.startsWith(column.getSimpleName())) {
+ types.put(column.getSimpleName(), column.getDataType().getType());
+ }
+ }
+ }
parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR);
}
+ private static String makePath(String [] path, int depth) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i <= depth; i++) {
+ sb.append(path[i]);
+ if (i < depth) {
+ sb.append(NestedPathUtil.PATH_DELIMITER);
+ }
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ *
+ *
+ * @param object
+ * @param pathElements
+ * @param depth
+ * @param fieldIndex
+ * @param output
+ * @throws IOException
+ */
+ private void getValue(JSONObject object,
+ String fullPath,
+ String [] pathElements,
+ int depth,
+ int fieldIndex,
+ Tuple output) throws IOException {
+ String fieldName = pathElements[depth];
+
+ if (!object.containsKey(fieldName)) {
+ output.put(fieldIndex, NullDatum.get());
+ }
+
+ switch (types.get(fullPath)) {
+ case BOOLEAN:
+ String boolStr = object.getAsString(fieldName);
+ if (boolStr != null) {
+ output.put(fieldIndex, DatumFactory.createBool(boolStr.equals("true")));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case CHAR:
+ String charStr = object.getAsString(fieldName);
+ if (charStr != null) {
+ output.put(fieldIndex, DatumFactory.createChar(charStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case INT1:
+ case INT2:
+ Number int2Num = object.getAsNumber(fieldName);
+ if (int2Num != null) {
+ output.put(fieldIndex, DatumFactory.createInt2(int2Num.shortValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case INT4:
+ Number int4Num = object.getAsNumber(fieldName);
+ if (int4Num != null) {
+ output.put(fieldIndex, DatumFactory.createInt4(int4Num.intValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case INT8:
+ Number int8Num = object.getAsNumber(fieldName);
+ if (int8Num != null) {
+ output.put(fieldIndex, DatumFactory.createInt8(int8Num.longValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case FLOAT4:
+ Number float4Num = object.getAsNumber(fieldName);
+ if (float4Num != null) {
+ output.put(fieldIndex, DatumFactory.createFloat4(float4Num.floatValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case FLOAT8:
+ Number float8Num = object.getAsNumber(fieldName);
+ if (float8Num != null) {
+ output.put(fieldIndex, DatumFactory.createFloat8(float8Num.doubleValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case TEXT:
+ String textStr = object.getAsString(fieldName);
+ if (textStr != null) {
+ output.put(fieldIndex, DatumFactory.createText(textStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case TIMESTAMP:
+ String timestampStr = object.getAsString(fieldName);
+ if (timestampStr != null) {
+ output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case TIME:
+ String timeStr = object.getAsString(fieldName);
+ if (timeStr != null) {
+ output.put(fieldIndex, DatumFactory.createTime(timeStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case DATE:
+ String dateStr = object.getAsString(fieldName);
+ if (dateStr != null) {
+ output.put(fieldIndex, DatumFactory.createDate(dateStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case BIT:
+ case BINARY:
+ case VARBINARY:
+ case BLOB: {
+ Object jsonObject = object.getAsString(fieldName);
+
+ if (jsonObject == null) {
+ output.put(fieldIndex, NullDatum.get());
+ break;
+ }
+
+ output.put(fieldIndex, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject)));
+ break;
+ }
+ case INET4:
+ String inetStr = object.getAsString(fieldName);
+ if (inetStr != null) {
+ output.put(fieldIndex, DatumFactory.createInet4(inetStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+
+ case RECORD:
+ JSONObject nestedObject = (JSONObject) object.get(fieldName);
+ if (nestedObject != null) {
+ getValue(nestedObject, fullPath + "/" + pathElements[depth+1], pathElements, depth + 1, fieldIndex, output);
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+
+ case NULL_TYPE:
+ output.put(fieldIndex, NullDatum.get());
+ break;
+
+ default:
+ throw new NotImplementedException(types.get(fullPath).name() + " is not supported.");
+ }
+ }
+
@Override
public void deserialize(ByteBuf buf, Tuple output) throws IOException, TextLineParsingError {
byte[] line = new byte[buf.readableBytes()];
@@ -70,135 +257,9 @@ public class JsonLineDeserializer extends TextLineDeserializer {
throw new TextLineParsingError(new String(line, TextDatum.DEFAULT_CHARSET), ae);
}
- for (int i = 0; i < targetColumnIndexes.length; i++) {
- int actualIdx = targetColumnIndexes[i];
- String fieldName = columnNames[actualIdx];
-
- if (!object.containsKey(fieldName)) {
- output.put(actualIdx, NullDatum.get());
- continue;
- }
-
- switch (types[actualIdx]) {
- case BOOLEAN:
- String boolStr = object.getAsString(fieldName);
- if (boolStr != null) {
- output.put(actualIdx, DatumFactory.createBool(boolStr.equals("true")));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case CHAR:
- String charStr = object.getAsString(fieldName);
- if (charStr != null) {
- output.put(actualIdx, DatumFactory.createChar(charStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case INT1:
- case INT2:
- Number int2Num = object.getAsNumber(fieldName);
- if (int2Num != null) {
- output.put(actualIdx, DatumFactory.createInt2(int2Num.shortValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case INT4:
- Number int4Num = object.getAsNumber(fieldName);
- if (int4Num != null) {
- output.put(actualIdx, DatumFactory.createInt4(int4Num.intValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case INT8:
- Number int8Num = object.getAsNumber(fieldName);
- if (int8Num != null) {
- output.put(actualIdx, DatumFactory.createInt8(int8Num.longValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case FLOAT4:
- Number float4Num = object.getAsNumber(fieldName);
- if (float4Num != null) {
- output.put(actualIdx, DatumFactory.createFloat4(float4Num.floatValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case FLOAT8:
- Number float8Num = object.getAsNumber(fieldName);
- if (float8Num != null) {
- output.put(actualIdx, DatumFactory.createFloat8(float8Num.doubleValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case TEXT:
- String textStr = object.getAsString(fieldName);
- if (textStr != null) {
- output.put(actualIdx, DatumFactory.createText(textStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case TIMESTAMP:
- String timestampStr = object.getAsString(fieldName);
- if (timestampStr != null) {
- output.put(actualIdx, DatumFactory.createTimestamp(timestampStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case TIME:
- String timeStr = object.getAsString(fieldName);
- if (timeStr != null) {
- output.put(actualIdx, DatumFactory.createTime(timeStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case DATE:
- String dateStr = object.getAsString(fieldName);
- if (dateStr != null) {
- output.put(actualIdx, DatumFactory.createDate(dateStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case BIT:
- case BINARY:
- case VARBINARY:
- case BLOB: {
- Object jsonObject = object.getAsString(fieldName);
-
- if (jsonObject == null) {
- output.put(actualIdx, NullDatum.get());
- break;
- }
-
- output.put(actualIdx, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject)));
- break;
- }
- case INET4:
- String inetStr = object.getAsString(fieldName);
- if (inetStr != null) {
- output.put(actualIdx, DatumFactory.createInet4(inetStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
-
- case NULL_TYPE:
- output.put(actualIdx, NullDatum.get());
- break;
-
- default:
- throw new NotImplementedException(types[actualIdx].name() + " is not supported.");
- }
+ for (int i = 0; i < projectedPaths.length; i++) {
+ String [] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER);
+ getValue(object, paths[0], paths, 0, i, output);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
index 6db2c29..5f12d76 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage.json;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.storage.text.TextLineDeserializer;
@@ -26,8 +27,8 @@ import org.apache.tajo.storage.text.TextLineSerializer;
public class JsonLineSerDe extends TextLineSerDe {
@Override
- public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- return new JsonLineDeserializer(schema, meta, targetColumnIndexes);
+ public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ return new JsonLineDeserializer(schema, meta, projected);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
index d6faf2d..34e9661 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
@@ -35,8 +35,6 @@ import java.io.IOException;
import java.io.OutputStream;
public class JsonLineSerializer extends TextLineSerializer {
- private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
-
private Type [] types;
private String [] simpleNames;
private int columnNum;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
index a091eac..4c675a4 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
@@ -77,10 +77,11 @@ public class TajoRecordConverter extends GroupConverter {
continue;
}
Type type = parquetSchema.getType(index);
+ final int writeIndex = i;
converters[index] = newConverter(column, type, new ParentValueContainer() {
@Override
void add(Object value) {
- TajoRecordConverter.this.set(projectionIndex, value);
+ TajoRecordConverter.this.set(writeIndex, value);
}
});
++index;
@@ -145,7 +146,7 @@ public class TajoRecordConverter extends GroupConverter {
*/
@Override
public void start() {
- currentTuple = new VTuple(tupleSize);
+ currentTuple = new VTuple(projectionMap.length);
}
/**
@@ -157,7 +158,7 @@ public class TajoRecordConverter extends GroupConverter {
final int projectionIndex = projectionMap[i];
Column column = tajoReadSchema.getColumn(projectionIndex);
if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE
- || currentTuple.get(projectionIndex) == null) {
+ || currentTuple.get(i) == null) {
set(projectionIndex, NullDatum.get());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
index 62e5ed9..af260b4 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
@@ -1635,7 +1635,7 @@ public class RCFile {
return null;
}
- Tuple tuple = new VTuple(schema.size());
+ Tuple tuple = new VTuple(targets.length);
getCurrentRow(tuple);
return tuple;
}
@@ -1705,16 +1705,16 @@ public class RCFile {
for (int j = 0; j < selectedColumns.length; ++j) {
SelectedColumn col = selectedColumns[j];
- int i = col.colIndex;
+ int actualColumnIdx = col.colIndex;
if (col.isNulled) {
- tuple.put(i, NullDatum.get());
+ tuple.put(j, NullDatum.get());
} else {
colAdvanceRow(j, col);
- Datum datum = serde.deserialize(schema.getColumn(i),
+ Datum datum = serde.deserialize(schema.getColumn(actualColumnIdx),
currentValue.loadedColumnsValueBuffer[j].getData(), col.rowReadIndex, col.prvLength, nullChars);
- tuple.put(i, datum);
+ tuple.put(j, datum);
col.rowReadIndex += col.prvLength;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
index 92a041c..af0973e 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
@@ -108,9 +108,9 @@ public class SequenceFileScanner extends FileScanner {
}
- fieldIsNull = new boolean[schema.getColumns().size()];
- fieldStart = new int[schema.getColumns().size()];
- fieldLength = new int[schema.getColumns().size()];
+ fieldIsNull = new boolean[schema.getRootColumns().size()];
+ fieldStart = new int[schema.getRootColumns().size()];
+ fieldLength = new int[schema.getRootColumns().size()];
prepareProjection(targets);
@@ -172,7 +172,7 @@ public class SequenceFileScanner extends FileScanner {
Text text = new Text();
reader.getCurrentValue(text);
cells = BytesUtils.splitPreserveAllTokens(text.getBytes(),
- delimiter, projectionMap, schema.getColumns().size());
+ delimiter, projectionMap, schema.getRootColumns().size());
totalBytes += (long)text.getBytes().length;
tuple = new LazyTuple(schema, cells, 0, nullChars, serde);
}
@@ -197,7 +197,7 @@ public class SequenceFileScanner extends FileScanner {
* So, tajo must make a tuple after parsing hive style BinarySerDe.
*/
private Tuple makeTuple(BytesWritable value) throws IOException{
- Tuple tuple = new VTuple(schema.getColumns().size());
+ Tuple tuple = new VTuple(schema.getRootColumns().size());
int start = 0;
int length = value.getLength();
@@ -213,7 +213,7 @@ public class SequenceFileScanner extends FileScanner {
int lastFieldByteEnd = start + 1;
// Go through all bytes in the byte[]
- for (int i = 0; i < schema.getColumns().size(); i++) {
+ for (int i = 0; i < schema.getRootColumns().size(); i++) {
fieldIsNull[i] = true;
if ((nullByte & (1 << (i % 8))) != 0) {
fieldIsNull[i] = false;
@@ -322,12 +322,12 @@ public class SequenceFileScanner extends FileScanner {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
public boolean isSelectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
index 03a0a26..0901c0b 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
@@ -19,6 +19,7 @@
package org.apache.tajo.storage.text;
import io.netty.buffer.ByteBuf;
+import org.apache.tajo.catalog.Column;
import io.netty.buffer.ByteBufProcessor;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -28,6 +29,7 @@ import org.apache.tajo.storage.FieldSerializerDeserializer;
import org.apache.tajo.storage.Tuple;
import java.io.IOException;
+import java.util.Arrays;
public class CSVLineDeserializer extends TextLineDeserializer {
private ByteBufProcessor processor;
@@ -35,8 +37,18 @@ public class CSVLineDeserializer extends TextLineDeserializer {
private ByteBuf nullChars;
private int delimiterCompensation;
- public CSVLineDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- super(schema, meta, targetColumnIndexes);
+ private int [] targetColumnIndexes;
+ private Column [] projected;
+
+ public CSVLineDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ super(schema, meta);
+
+ this.projected = projected;
+ targetColumnIndexes = new int[projected.length];
+ for (int i = 0; i < projected.length; i++) {
+ targetColumnIndexes[i] = schema.getColumnId(projected[i].getQualifiedName());
+ }
+ Arrays.sort(targetColumnIndexes);
}
@Override
@@ -66,7 +78,7 @@ public class CSVLineDeserializer extends TextLineDeserializer {
final int rowLength = lineBuf.readableBytes();
int start = 0, fieldLength = 0, end = 0;
- //Projection
+ // Projection
int currentTarget = 0;
int currentIndex = 0;
@@ -83,10 +95,10 @@ public class CSVLineDeserializer extends TextLineDeserializer {
lineBuf.setIndex(start, start + fieldLength);
try {
- Datum datum = fieldSerDer.deserialize(lineBuf, schema.getColumn(currentIndex), currentIndex, nullChars);
- output.put(currentIndex, datum);
+ Datum datum = fieldSerDer.deserialize(lineBuf, projected[currentTarget], currentIndex, nullChars);
+ output.put(currentTarget, datum);
} catch (Exception e) {
- output.put(currentIndex, NullDatum.get());
+ output.put(currentTarget, NullDatum.get());
}
currentTarget++;
@@ -103,7 +115,7 @@ public class CSVLineDeserializer extends TextLineDeserializer {
/* If a text row is less than table schema size, tuple should set to NullDatum */
if (projection.length > currentTarget) {
for (; currentTarget < projection.length; currentTarget++) {
- output.put(projection[currentTarget], NullDatum.get());
+ output.put(currentTarget, NullDatum.get());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
index 988d5d1..4ebdbe8 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
@@ -19,6 +19,7 @@
package org.apache.tajo.storage.text;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.storage.StorageConstants;
@@ -26,8 +27,8 @@ import org.apache.tajo.util.Bytes;
public class CSVLineSerDe extends TextLineSerDe {
@Override
- public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- return new CSVLineDeserializer(schema, meta, targetColumnIndexes);
+ public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ return new CSVLineDeserializer(schema, meta, projected);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
index 5e7bd94..55a2b96 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
@@ -275,7 +275,6 @@ public class DelimitedTextFile {
private final long endOffset;
/** The number of actual read records */
private int recordCount = 0;
- private int[] targetColumnIndexes;
private DelimitedLineReader reader;
private TextLineDeserializer deserializer;
@@ -321,13 +320,7 @@ public class DelimitedTextFile {
targets = schema.toArray();
}
- targetColumnIndexes = new int[targets.length];
- for (int i = 0; i < targets.length; i++) {
- targetColumnIndexes[i] = schema.getColumnId(targets[i].getQualifiedName());
- }
-
super.init();
- Arrays.sort(targetColumnIndexes);
if (LOG.isDebugEnabled()) {
LOG.debug("DelimitedTextFileScanner open:" + fragment.getPath() + "," + startOffset + "," + endOffset);
}
@@ -336,7 +329,7 @@ public class DelimitedTextFile {
reader.readLine(); // skip first line;
}
- deserializer = getLineSerde().createDeserializer(schema, meta, targetColumnIndexes);
+ deserializer = getLineSerde().createDeserializer(schema, meta, targets);
deserializer.init();
}
@@ -391,7 +384,7 @@ public class DelimitedTextFile {
return EmptyTuple.get();
}
- tuple = new VTuple(schema.size());
+ tuple = new VTuple(targets.length);
tuple.setOffset(offset);
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
index 89a7de9..f067cb3 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
@@ -31,12 +31,10 @@ import java.io.IOException;
public abstract class TextLineDeserializer {
protected final Schema schema;
protected final TableMeta meta;
- protected final int[] targetColumnIndexes;
- public TextLineDeserializer(Schema schema, TableMeta meta, int [] targetColumnIndexes) {
+ public TextLineDeserializer(Schema schema, TableMeta meta) {
this.schema = schema;
this.meta = meta;
- this.targetColumnIndexes = targetColumnIndexes;
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
index 1a53bb0..c09a83b 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
@@ -21,6 +21,7 @@ package org.apache.tajo.storage.text;
import io.netty.buffer.ByteBuf;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.datum.NullDatum;
@@ -36,7 +37,7 @@ public abstract class TextLineSerDe {
public TextLineSerDe() {
}
- public abstract TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int [] targetColumnIndexes);
+ public abstract TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, Column [] projected);
public abstract TextLineSerializer createSerializer(Schema schema, TableMeta meta);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index 561e2ef..322818d 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@ -175,7 +175,19 @@ public class TestMergeScanner {
Tuple tuple;
while ((tuple = scanner.next()) != null) {
totalCounts++;
- if (isProjectableStorage(meta.getStoreType())) {
+
+ if (storeType == StoreType.RAW) {
+ assertEquals(4, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ assertNotNull(tuple.get(2));
+ assertNotNull(tuple.get(3));
+ } else if (scanner.isProjectable()) {
+ assertEquals(2, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ } else {
+ assertEquals(4, tuple.size());
assertNotNull(tuple.get(0));
assertNull(tuple.get(1));
assertNotNull(tuple.get(2));
@@ -189,14 +201,13 @@ public class TestMergeScanner {
private static boolean isProjectableStorage(StoreType type) {
switch (type) {
- case RCFILE:
- case PARQUET:
- case SEQUENCEFILE:
- case CSV:
- case AVRO:
- return true;
- default:
- return false;
+ case CSV:
+ case SEQUENCEFILE:
+ case RAW:
+ case ROWFILE:
+ return false;
+ default:
+ return true;
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 456ea00..a735307 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -297,15 +297,7 @@ public class TestStorages {
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
- if (storeType == StoreType.RCFILE
- || storeType == StoreType.CSV
- || storeType == StoreType.PARQUET
- || storeType == StoreType.SEQUENCEFILE
- || storeType == StoreType.AVRO) {
- assertTrue(tuple.get(0) == null);
- }
- assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
- assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
+ verifyProjectedFields(scanner.isProjectable(), tuple, tupleCnt);
tupleCnt++;
}
scanner.close();
@@ -313,6 +305,20 @@ public class TestStorages {
assertEquals(tupleNum, tupleCnt);
}
+ private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCnt) {
+ if (projectable) {
+ assertTrue(tupleCnt + 2 == tuple.get(0).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(1).asFloat4());
+ } else {
+ // RAW and ROW always project all fields.
+ if (storeType != StoreType.RAW && storeType != StoreType.ROWFILE) {
+ assertTrue(tuple.get(0) == null);
+ }
+ assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
+ }
+ }
+
@Test
public void testVariousTypes() throws IOException {
boolean handleProtobuf = storeType != StoreType.JSON;
@@ -956,7 +962,7 @@ public class TestStorages {
@Test
public void testLessThanSchemaSize() throws IOException {
/* RAW is internal storage. It must be same with schema size */
- if (storeType == StoreType.RAW || storeType == StoreType.AVRO){
+ if (storeType == StoreType.RAW || storeType == StoreType.AVRO || storeType == StoreType.PARQUET) {
return;
}
@@ -1008,7 +1014,12 @@ public class TestStorages {
Tuple tuple = scanner.next();
scanner.close();
- assertEquals(expect.get(1), tuple.get(1));
- assertEquals(NullDatum.get(), tuple.get(4));
+ if (scanner.isProjectable()) {
+ assertEquals(expect.get(1), tuple.get(0));
+ assertEquals(NullDatum.get(), tuple.get(1));
+ } else {
+ assertEquals(expect.get(1), tuple.get(1));
+ assertEquals(NullDatum.get(), tuple.get(4));
+ }
}
}
[11/14] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
new file mode 100644
index 0000000..e750095
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
@@ -0,0 +1,74 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS ${0} (
+ coordinates TEXT,
+ favorited BOOL,
+ truncated BOOL,
+ created_at TIMESTAMP,
+ id_str TEXT,
+ /*entrities RECORD ( -- when we support array, we should remove this comment.
+ urls ARRAY<TEXT>
+ )*/
+ in_reply_to_user_id_str TEXT,
+ contributors TEXT,
+ text TEXT,
+ metadata RECORD (
+ iso_language_code TEXT,
+ result_type TEXT
+ ),
+ retweet_count INTEGER,
+ in_reply_to_status_id_str TEXT,
+ id TEXT,
+ geo TEXT,
+ retweeted BOOL,
+ in_reply_to_user_id TEXT,
+ place TEXT,
+ user RECORD (
+ profile_sidebar_fill_color TEXT,
+ profile_sidebar_border_color TEXT,
+ profile_background_tile TEXT,
+ name TEXT,
+ profile_image_url TEXT,
+ created_at TIMESTAMP,
+ location TEXT,
+ follow_request_sent TEXT,
+ profile_link_color TEXT,
+ is_translator BOOL,
+ id_str TEXT,
+ /* -- when we support array, we should fill the following comments.
+ entities RECORD (
+ url RECORD (
+ ),
+ description RECORD (
+ )
+ ), */
+ default_profile BOOL,
+ contributors_enabled BOOL,
+ favourites_count INTEGER,
+ url TEXT,
+ profile_image_url_https TEXT,
+ utc_offset INTEGER,
+ id BIGINT,
+ profile_use_background_image BOOL,
+ listed_count INTEGER,
+ profile_text_color TEXT,
+ lang TEXT,
+ followers_count INTEGER,
+ protected BOOL,
+ notifications TEXT,
+ profile_background_image_url_https TEXT,
+ profile_background_color TEXT,
+ verified TEXT,
+ geo_enabled TEXT,
+ time_zone TEXT,
+ description TEXT,
+ default_profile_image TEXT,
+ profile_background_image_url TEXT,
+ statuses_count INTEGER,
+ friends_count INTEGER,
+ following TEXT,
+ show_all_inline_media BOOL,
+ screen_name TEXT
+ ),
+ in_reply_to_screen_name TEXT,
+ source TEXT,
+ in_reply_to_status_id TEXT
+) USING JSON LOCATION ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
index 1760c99..e707a8c 100644
--- a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
@@ -1 +1 @@
-(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n_nationkey = 1);
+(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n1.n_nationkey = 1);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
new file mode 100644
index 0000000..debf06e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
@@ -0,0 +1,6 @@
+user/name,total_retweet
+-------------------------------
+Chaz Martenstein,2
+Thomas John Wakeman,3
+Sean Cummings,1
+Marty Elmer,4
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
new file mode 100644
index 0000000..1c57dc2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
@@ -0,0 +1,6 @@
+user/id,user/name,user/id,user/name
+-------------------------------
+137238150,Sean Cummings,137238150,Sean Cummings
+29516238,Chaz Martenstein,29516238,Chaz Martenstein
+70789458,Thomas John Wakeman,70789458,Thomas John Wakeman
+37539828,Marty Elmer,37539828,Marty Elmer
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
new file mode 100644
index 0000000..f96fcc5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
@@ -0,0 +1,5 @@
+title,full_name
+-------------------------------
+Hand of the King,Eddard Stark
+Assassin,Arya Stark
+Dancing Master,Syrio Forel
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
new file mode 100644
index 0000000..90eadbd
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
@@ -0,0 +1,6 @@
+coordinates,favorited,truncated,created_at,id_str,in_reply_to_user_id_str,contributors,text,metadata/iso_language_code,metadata/result_type,retweet_count,in_reply_to_status_id_str,id,geo,retweeted,in_reply_to_user_id,place,user/profile_sidebar_fill_color,user/profile_sidebar_border_color,user/profile_background_tile,user/name,user/profile_image_url,user/created_at,user/location,user/follow_request_sent,user/profile_link_color,user/is_translator,user/id_str,user/default_profile,user/contributors_enabled,user/favourites_count,user/url,user/profile_image_url_https,user/utc_offset,user/id,user/profile_use_background_image,user/listed_count,user/profile_text_color,user/lang,user/followers_count,user/protected,user/notifications,user/profile_background_image_url_https,user/profile_background_color,user/verified,user/geo_enabled,user/time_zone,user/description,user/default_profile_image,user/profile_background_image_url,user/statuses_count,user/friends_count,user/following,user/show_all_in
line_media,user/screen_name,in_reply_to_screen_name,source,in_reply_to_status_id
+-------------------------------
+null,false,false,2012-09-24 03:35:21,250075927172759552,null,null,Aggressive Ponytail #freebandnames,en,recent,1,null,250075927172759552,null,false,null,null,DDEEF6,C0DEED,false,Sean Cummings,http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,2010-04-26 06:01:55,LA, CA,null,0084B4,false,137238150,true,false,0,null,https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,-28800,137238150,true,2,333333,en,70,false,null,https://si0.twimg.com/images/themes/theme1/bg.png,C0DEED,false,true,Pacific Time (US & Canada),Born 330 Live 310,false,http://a0.twimg.com/images/themes/theme1/bg.png,579,110,null,false,sean_cummings,null,<a>Twitter for Mac</a>,null
+null,false,false,2012-09-21 23:40:54,249292149810667520,null,null,Thee Namaste Nerdz. #FreeBandNames,pl,recent,2,null,249292149810667520,null,false,null,null,DDFFCC,BDDCAD,true,Chaz Martenstein,http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,2009-04-07 19:05:07,Durham, NC,null,0084B4,false,29516238,false,false,8,http://bullcityrecords.com/wnng/,https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,-18000,29516238,true,118,333333,en,2052,false,null,https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp,9AE4E8,false,false,Eastern Time (US & Canada),You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.,false,http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp,7579,348,null,true,bullcityrecords,null,web,null
+null,false,false,2012-09-21 23:30:20,249289491129438208,null,null,Mexican Heaven, Mexican Hell #freebandnames,en,recent,3,null,249289491129438208,null,false,null,null,99CC33,829D5E,false,Thomas John Wakeman,http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,2009-09-01 21:21:35,Kingston New York,null,D02B55,false,70789458,false,false,19,null,https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,-18000,70789458,true,1,3E4415,en,63,false,null,https://si0.twimg.com/images/themes/theme5/bg.gif,352726,false,false,Eastern Time (US & Canada),Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.,false,http://a0.twimg.com/images/themes/theme5/bg.gif,1048,63,null,false,MonkiesFist,null,web,null
+null,false,false,2012-09-21 22:51:18,249279667666817024,null,null,The Foolish Mortals #freebandnames,en,recent,4,null,249279667666817024,null,false,null,null,BFAC83,615A44,true,Marty Elmer,http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,2009-05-04 00:05:00,Wisconsin, USA,null,3B2A26,false,37539828,false,false,647,http://www.omnitarian.me,https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,-21600,37539828,true,52,000000,en,608,false,null,https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png,EEE3C4,false,false,Central Time (US & Canada),Cartoonist, Illustrator, and T-Shirt connoisseur,false,http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png,3575,249,null,true,Omnitarian,null,<a>Twitter for iPhone</a>,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
index 2368082..d4ef55e 100644
--- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
+++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
@@ -513,7 +513,7 @@ public class TajoDatabaseMetaData implements DatabaseMetaData {
CatalogUtil.buildFQName(databaseName, table));
int pos = 0;
- for (Column column: tableDesc.getLogicalSchema().getColumns()) {
+ for (Column column: tableDesc.getLogicalSchema().getRootColumns()) {
if (column.getSimpleName().matches(regcolumnNamePattern)) {
MetaDataTuple tuple = new MetaDataTuple(22);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
index 6262995..fbad76e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
@@ -114,7 +114,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
Schema schema = relationOp.getLogicalSchema();
Column[] resolvedColumns = new Column[schema.size()];
- return schema.getColumns().toArray(resolvedColumns);
+ return schema.getRootColumns().toArray(resolvedColumns);
} else { // if a column reference is not qualified
// columns of every relation should be resolved.
Iterator<RelationNode> iterator = block.getRelations().iterator();
@@ -124,7 +124,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
while (iterator.hasNext()) {
relationOp = iterator.next();
schema = relationOp.getLogicalSchema();
- resolvedColumns.addAll(schema.getColumns());
+ resolvedColumns.addAll(schema.getRootColumns());
}
if (resolvedColumns.size() == 0) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index e0b4f7e..21270e9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -52,6 +52,7 @@ import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -527,13 +528,24 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
} else if (projectable instanceof RelationNode) {
RelationNode relationNode = (RelationNode) projectable;
+ prohibitNestedRecordProjection((Projectable) relationNode);
verifyIfTargetsCanBeEvaluated(relationNode.getLogicalSchema(), (Projectable) relationNode);
} else {
+ prohibitNestedRecordProjection(projectable);
verifyIfTargetsCanBeEvaluated(projectable.getInSchema(), projectable);
}
}
+ public static void prohibitNestedRecordProjection(Projectable projectable)
+ throws PlanningException {
+ for (Target t : projectable.getTargets()) {
+ if (t.getEvalTree().getValueType().getType() == TajoDataTypes.Type.RECORD) {
+ throw new PlanningException("Projecting RECORD fields is not supported yet: " + t);
+ }
+ }
+ }
+
public static void verifyIfEvalNodesCanBeEvaluated(Projectable projectable, EvalNode[] evalNodes)
throws PlanningException {
for (EvalNode e : evalNodes) {
@@ -649,7 +661,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
if (block.namedExprsMgr.isEvaluated(sortKeyRefNames[i])) {
column = block.namedExprsMgr.getTarget(sortKeyRefNames[i]).getNamedColumn();
} else {
- throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs));
+ throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs));
}
annotatedSortSpecs[i] = new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst());
}
@@ -866,7 +878,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
} else if (block.namedExprsMgr.isEvaluated(refName)) {
column = block.namedExprsMgr.getTarget(refName).getNamedColumn();
} else {
- throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs));
+ throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs));
}
annotatedSortSpecs.add(new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst()));
}
@@ -1175,13 +1187,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Schema joinSchema = new Schema();
Schema commons = SchemaUtil.getNaturalJoinColumns(left.getOutSchema(), right.getOutSchema());
joinSchema.addColumns(commons);
- for (Column c : left.getOutSchema().getColumns()) {
+ for (Column c : left.getOutSchema().getRootColumns()) {
if (!joinSchema.contains(c.getQualifiedName())) {
joinSchema.addColumn(c);
}
}
- for (Column c : right.getOutSchema().getColumns()) {
+ for (Column c : right.getOutSchema().getRootColumns()) {
if (!joinSchema.contains(c.getQualifiedName())) {
joinSchema.addColumn(c);
}
@@ -1199,7 +1211,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Column leftJoinKey;
Column rightJoinKey;
- for (Column common : commons.getColumns()) {
+ for (Column common : commons.getRootColumns()) {
leftJoinKey = leftSchema.getColumn(common.getQualifiedName());
rightJoinKey = rightSchema.getColumn(common.getQualifiedName());
equiQual = new BinaryEval(EvalType.EQUAL,
@@ -1312,7 +1324,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
private static LinkedHashSet<Target> createFieldTargetsFromRelation(QueryBlock block, RelationNode relationNode,
Set<String> newlyEvaluatedRefNames) {
LinkedHashSet<Target> targets = Sets.newLinkedHashSet();
- for (Column column : relationNode.getLogicalSchema().getColumns()) {
+ for (Column column : relationNode.getLogicalSchema().getAllColumns()) {
+
+ // TODO - Currently, EvalNode has DataType as a return type. So, RECORD cannot be used for any target.
+ // The following line is a kind of hack, preventing RECORD to be used for target in the logical planning phase.
+ // This problem should be resolved after TAJO-1402.
+ if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) {
+ continue;
+ }
+
String aliasName = block.namedExprsMgr.checkAndGetIfAliasedColumn(column.getQualifiedName());
if (aliasName != null) {
targets.add(new Target(new FieldEval(column), aliasName));
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
index 153a150..eddb022 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
@@ -26,6 +26,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import static org.apache.tajo.common.TajoDataTypes.DataType;
@@ -88,7 +89,7 @@ public class RowConstantEval extends EvalNode {
}
public String toString() {
- return TUtil.arrayToString(values);
+ return StringUtils.join(values);
}
public void preOrder(EvalNodeVisitor visitor) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
index 0b60d14..a39d303 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
@@ -26,6 +26,7 @@ import org.apache.tajo.datum.Datum;
import org.apache.tajo.plan.function.FunctionContext;
import org.apache.tajo.plan.logical.WindowSpec;
import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.Arrays;
@@ -120,7 +121,7 @@ public class WindowFunctionEval extends AggregationFunctionCallEval implements C
sb.append(funcDesc.getFunctionName()).append("(").append(isDistinct() ? " distinct" : "").append(sb)
.append(")");
if (hasSortSpecs()) {
- sb.append("ORDER BY ").append(TUtil.arrayToString(sortSpecs));
+ sb.append("ORDER BY ").append(StringUtils.join(sortSpecs));
}
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
index 878553f..079fc3e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
@@ -478,7 +478,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
@@ -512,7 +512,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -532,7 +532,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -552,7 +552,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext(functionContext);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -573,7 +573,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
return outputHandler.getPartialResultString();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -595,7 +595,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
index e4d6122..862cb8a 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
@@ -26,7 +26,7 @@ import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.expr.AlgebraicUtil;
import org.apache.tajo.plan.logical.*;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.*;
@@ -59,7 +59,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
for (LogicalNode relation : remainRelations) {
Collection <String> relationStrings = PlannerUtil.getRelationLineageWithinQueryBlock(plan, relation);
List<JoinEdge> joinEdges = new ArrayList<JoinEdge>();
- String relationCollection = TUtil.collectionToString(relationStrings, ",");
+ String relationCollection = StringUtils.join(relationStrings, ",");
List<JoinEdge> joinEdgesForGiven = joinGraph.getIncomingEdges(relationCollection);
if (joinEdgesForGiven != null) {
joinEdges.addAll(joinEdgesForGiven);
@@ -236,7 +236,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
// If outer is outer join, make edge key using all relation names in outer.
SortedSet<String> relationNames =
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer));
- String outerEdgeKey = TUtil.collectionToString(relationNames, ", ");
+ String outerEdgeKey = StringUtils.join(relationNames, ", ");
for (String innerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner)) {
if (graph.hasEdge(outerEdgeKey, innerName)) {
JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, innerName);
@@ -257,7 +257,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
relationNames =
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner));
- outerEdgeKey = TUtil.collectionToString(relationNames, ", ");
+ outerEdgeKey = StringUtils.join(relationNames, ", ");
for (String outerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer)) {
if (graph.hasEdge(outerEdgeKey, outerName)) {
JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, outerName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
index ce06748..fb4fae1 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
@@ -22,7 +22,7 @@ import com.google.common.collect.Sets;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.Collections;
import java.util.Set;
@@ -70,6 +70,6 @@ public class JoinEdge {
}
public String toString() {
- return leftRelation + " " + joinType + " " + rightRelation + " ON " + TUtil.collectionToString(joinQual, ", ");
+ return leftRelation + " " + joinType + " " + rightRelation + " ON " + StringUtils.join(joinQual, ", ");
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
index 72e9b1d..9ae5245 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Sets;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.graph.SimpleUndirectedGraph;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.NamedExprsManager;
@@ -33,7 +34,6 @@ import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.expr.EvalTreeUtil;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.plan.logical.RelationNode;
-import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -111,8 +111,8 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> {
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, joinNode.getRightChild()));
addEdge(
- TUtil.collectionToString(leftNodeRelationName, ", "),
- TUtil.collectionToString(rightNodeRelationName, ", "),
+ StringUtils.join(leftNodeRelationName, ", "),
+ StringUtils.join(rightNodeRelationName, ", "),
edge);
Set<EvalNode> allInOneCnf = new HashSet<EvalNode>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
index 61ce789..fb19f10 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
@@ -19,12 +19,12 @@
package org.apache.tajo.plan.logical;
import com.google.gson.annotations.Expose;
-
import org.apache.tajo.catalog.Column;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.AggregationFunctionCallEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.ArrayList;
@@ -150,7 +150,7 @@ public class DistinctGroupbyNode extends UnaryNode implements Projectable, Clone
public String toString() {
StringBuilder sb = new StringBuilder("Distinct GroupBy (");
if (groupingColumns != null && groupingColumns.length > 0) {
- sb.append("grouping set=").append(TUtil.arrayToString(groupingColumns));
+ sb.append("grouping set=").append(StringUtils.join(groupingColumns));
sb.append(", ");
}
for (GroupbyNode eachNode: subGroupbyPlan) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
index 3aaf5d0..0632f68 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
@@ -28,6 +28,7 @@ import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class EvalExprNode extends LogicalNode implements Projectable {
@@ -69,7 +70,7 @@ public class EvalExprNode extends LogicalNode implements Projectable {
@Override
public String toString() {
- return "EvalExprNode (" + TUtil.arrayToString(exprs) + ")";
+ return "EvalExprNode (" + StringUtils.join(exprs) + ")";
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
index 730f705..23a9154 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
@@ -18,18 +18,18 @@
package org.apache.tajo.plan.logical;
-import java.util.Arrays;
-
import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
-
import org.apache.tajo.catalog.Column;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.AggregationFunctionCallEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
+import java.util.Arrays;
+
public class GroupbyNode extends UnaryNode implements Projectable, Cloneable {
/** Grouping key sets */
@Expose private Column [] groupingKeys = PlannerUtil.EMPTY_COLUMNS;
@@ -110,11 +110,11 @@ public class GroupbyNode extends UnaryNode implements Projectable, Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("GroupBy (");
if (groupingKeys != null && groupingKeys.length > 0) {
- sb.append("grouping set=").append(TUtil.arrayToString(groupingKeys));
+ sb.append("grouping set=").append(StringUtils.join(groupingKeys));
sb.append(", ");
}
if (hasAggFunctions()) {
- sb.append("funcs=").append(TUtil.arrayToString(aggrFunctions));
+ sb.append("funcs=").append(StringUtils.join(aggrFunctions));
}
sb.append(")");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
index 3c9d497..ef6734e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
@@ -25,6 +25,7 @@ import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class ProjectionNode extends UnaryNode implements Projectable {
@@ -70,7 +71,7 @@ public class ProjectionNode extends UnaryNode implements Projectable {
public String toString() {
StringBuilder sb = new StringBuilder("Projection (distinct=").append(distinct);
if (targets != null) {
- sb.append(", exprs=").append(TUtil.arrayToString(targets)).append(")");
+ sb.append(", exprs=").append(StringUtils.join(targets)).append(")");
}
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
index 7e335b0..ced9a36 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
@@ -43,7 +43,19 @@ public abstract class RelationNode extends LogicalNode {
public abstract String getTableName();
+ /**
+ * Return a full qualified table name (i.e., dbname.table_name)
+ *
+ * @return A full qualified table name
+ */
public abstract String getCanonicalName();
+ /**
+ * Return a logical schema, meaning physically stored columns and virtual columns.
+ * Since partition keys in the column partition are not physically stored to files or tables,
+ * we call the partition keys virtual columns.
+ *
+ * @return A logical schema
+ */
public abstract Schema getLogicalSchema();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
index 8215f51..7487009 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
@@ -107,7 +108,7 @@ public class ShuffleFileWriteNode extends PersistentStoreNode implements Cloneab
}
sb.append(", part number=").append(numOutputs);
if (shuffleKeys != null) {
- sb.append(", keys: ").append(TUtil.arrayToString(shuffleKeys));
+ sb.append(", keys: ").append(StringUtils.join(shuffleKeys));
}
sb.append(")");
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
index 0166ef8..ef61d51 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
@@ -20,7 +20,7 @@ package org.apache.tajo.plan.logical;
import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.List;
@@ -52,7 +52,7 @@ public class TruncateTableNode extends LogicalNode {
@Override
public String toString() {
- return "TruncateTable (table=" + TUtil.collectionToString(tableNames, ", ") + ")";
+ return "TruncateTable (table=" + StringUtils.join(tableNames, ", ") + ")";
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
index 3f624f6..1bee34c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
@@ -26,6 +26,7 @@ import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.WindowFunctionEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class WindowAggNode extends UnaryNode implements Projectable, Cloneable {
@@ -114,14 +115,14 @@ public class WindowAggNode extends UnaryNode implements Projectable, Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("WinAgg (");
if (hasPartitionKeys()) {
- sb.append("partition keys=").append(TUtil.arrayToString(partitionKeys));
+ sb.append("partition keys=").append(StringUtils.join(partitionKeys));
sb.append(", ");
}
if (hasAggFunctions()) {
- sb.append("funcs=").append(TUtil.arrayToString(windowFuncs));
+ sb.append("funcs=").append(StringUtils.join(windowFuncs));
}
if (hasSortSpecs()) {
- sb.append("sort=").append(TUtil.arrayToString(sortSpecs));
+ sb.append("sort=").append(StringUtils.join(sortSpecs));
}
sb.append(")");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
index 2c55c81..633b30e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
@@ -18,11 +18,14 @@
package org.apache.tajo.plan.nameresolver;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.algebra.Relation;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.NestedPathUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.exception.NoSuchColumnException;
import org.apache.tajo.plan.algebra.AmbiguousFieldException;
@@ -31,14 +34,30 @@ import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.plan.logical.RelationNode;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
-import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
- * NameResolver utility
+ * Column name resolution utility. A SQL statement can include many kinds of column names,
+ * defined in different ways. Some column name indicates just a column in a relation.
+ * Another column name includes alias table name or alias column name, derived from some expression.
+ *
+ * This utility ensures that each column name is derived from valid and accessible column, and
+ * it also finds the exact data type of the column.
+ *
+ * Terminology:
+ * <ul>
+ * <li>Qualifier: database name, table name, or both included in a column name</li>
+ * <li>Simple name: just column name without any qualifier</li>
+ * <li>Alias name: another name to shortly specify a certain column</li>
+ * <li>Fully qualified name: a column name with database name and table name</li>
+ * <li>Canonical name: a fully qualified name, but its simple name is aliased name.</li>
+ * </ul>
*/
public abstract class NameResolver {
@@ -51,30 +70,44 @@ public abstract class NameResolver {
resolverMap.put(NameResolvingMode.LEGACY, new ResolverByLegacy());
}
+ public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
+ NameResolvingMode mode) throws PlanningException {
+ if (!resolverMap.containsKey(mode)) {
+ throw new PlanningException("Unsupported name resolving level: " + mode.name());
+ }
+ return resolverMap.get(mode).resolve(plan, block, column);
+ }
+
abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
throws PlanningException;
/**
- * Try to find the database name
+ * Guess a relation from a table name regardless of whether the given name is qualified or not.
*
* @param block the current block
- * @param tableName The table name
- * @return The found database name
+ * @param tableName The table name which can be either qualified or not.
+ * @return A corresponding relation
* @throws PlanningException
*/
- public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException {
- List<String> found = new ArrayList<String>();
+ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tableName) throws PlanningException {
+ List<RelationNode> found = TUtil.newList();
+
for (RelationNode relation : block.getRelations()) {
- // check alias name or table name
- if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
+
+ // if a table name is qualified
+ if (relation.getCanonicalName().equals(tableName) || relation.getTableName().equals(tableName)) {
+ found.add(relation);
+
+ // if a table name is not qualified
+ } else if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) {
- // obtain the database name
- found.add(CatalogUtil.extractQualifier(relation.getTableName()));
+ found.add(relation);
}
}
if (found.size() == 0) {
return null;
+
} else if (found.size() > 1) {
throw new PlanningException("Ambiguous table name \"" + tableName + "\"");
}
@@ -82,12 +115,26 @@ public abstract class NameResolver {
return found.get(0);
}
- public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
- NameResolvingMode mode) throws PlanningException {
- if (!resolverMap.containsKey(mode)) {
- throw new PlanningException("Unsupported name resolving level: " + mode.name());
+ /**
+ * Find relations such that its schema contains a given column
+ *
+ * @param block the current block
+ * @param columnName The column name to find relation
+ * @return relations including a given column
+ * @throws PlanningException
+ */
+ public static Collection<RelationNode> lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName)
+ throws PlanningException {
+
+ Set<RelationNode> found = TUtil.newHashSet();
+
+ for (RelationNode rel : block.getRelations()) {
+ if (rel.getLogicalSchema().contains(columnName)) {
+ found.add(rel);
+ }
}
- return resolverMap.get(mode).resolve(plan, block, column);
+
+ return found;
}
/**
@@ -107,7 +154,7 @@ public abstract class NameResolver {
String canonicalName;
if (columnRef.hasQualifier()) {
- Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef);
qualifier = normalized.getFirst();
canonicalName = normalized.getSecond();
@@ -121,8 +168,8 @@ public abstract class NameResolver {
// Please consider a query case:
// select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey;
//
- // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still can be used.
- // The below code makes it available. Otherwise, it cannot find any match in the relation schema.
+ // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still should be available.
+ // The below code makes it possible. Otherwise, it cannot find any match in the relation schema.
if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) {
canonicalName =
CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName));
@@ -133,7 +180,7 @@ public abstract class NameResolver {
return column;
} else {
- return resolveFromAllRelsInBlock(block, columnRef);
+ return lookupColumnFromAllRelsInBlock(block, columnRef.getName());
}
}
@@ -162,18 +209,22 @@ public abstract class NameResolver {
}
/**
- * It tries to find a full qualified column name from all relations in the current block.
+ * Lookup a column among all relations in the current block from a column name.
+ *
+ * It assumes that <code>columnName</code> is not any qualified name.
*
* @param block The current query block
- * @param columnRef The column reference to be found
+ * @param columnName The column reference to be found
* @return The found column
*/
- static Column resolveFromAllRelsInBlock(LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef) throws VerifyException {
+ static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block,
+ String columnName) throws VerifyException {
+ Preconditions.checkArgument(CatalogUtil.isSimpleIdentifier(columnName));
+
List<Column> candidates = TUtil.newList();
for (RelationNode rel : block.getRelations()) {
- Column found = rel.getLogicalSchema().getColumn(columnRef.getName());
+ Column found = rel.getLogicalSchema().getColumn(columnName);
if (found != null) {
candidates.add(found);
}
@@ -240,39 +291,100 @@ public abstract class NameResolver {
}
/**
- * It returns a pair of names, which the first value is ${database}.${table} and the second value
- * is a simple column name.
+ * Lookup a qualifier and a canonical name of column.
+ *
+ * It returns a pair of names, which the first value is the qualifier ${database}.${table} and
+ * the second value is column's simple name.
*
* @param block The current block
* @param columnRef The column name
* @return A pair of normalized qualifier and column name
* @throws PlanningException
*/
- static Pair<String, String> normalizeQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef)
+ static Pair<String, String> lookupQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef)
throws PlanningException {
- String qualifier;
- String canonicalName;
+ Preconditions.checkArgument(columnRef.hasQualifier(), "ColumnReferenceExpr must be qualified.");
+
+ String [] qualifierParts = columnRef.getQualifier().split("\\.");
- if (CatalogUtil.isFQTableName(columnRef.getQualifier())) {
- qualifier = columnRef.getQualifier();
- canonicalName = columnRef.getCanonicalName();
+ // This method assumes that column name consists of two or more dot chained names.
+ // In this case, there must be three cases as follows:
+ //
+ // - dbname.tbname.column_name.nested_field...
+ // - tbname.column_name.nested_field...
+ // - column.nested_fieldX...
+
+ Set<RelationNode> guessedRelations = TUtil.newHashSet();
+
+ // this position indicates the index of column name in qualifierParts;
+ // It must be 0 or more because a qualified column is always passed to lookupQualifierAndCanonicalName().
+ int columnNamePosition = -1;
+
+ // check for dbname.tbname.column_name.nested_field
+ if (qualifierParts.length >= 2) {
+ RelationNode rel = lookupTable(block, CatalogUtil.buildFQName(qualifierParts[0], qualifierParts[1]));
+ if (rel != null) {
+ guessedRelations.add(rel);
+ columnNamePosition = 2;
+ }
+ }
+
+ // check for tbname.column_name.nested_field
+ if (qualifierParts.length >= 1) {
+ RelationNode rel = lookupTable(block, qualifierParts[0]);
+ if (rel != null) {
+ guessedRelations.add(rel);
+ columnNamePosition = 1;
+ }
+ }
+
+ // column.nested_fieldX...
+ if (guessedRelations.size() == 0 && qualifierParts.length == 1) {
+ Collection<RelationNode> rels = lookupTableByColumns(block, qualifierParts[0]);
+
+ if (rels.size() > 1) {
+ throw new AmbiguousFieldException(columnRef.getCanonicalName());
+ }
+
+ if (rels.size() == 1) {
+ guessedRelations.addAll(rels);
+ columnNamePosition = 0;
+ }
+ }
+
+ // throw exception if no column cannot be founded or two or more than columns are founded
+ if (guessedRelations.size() == 0) {
+ throw new NoSuchColumnException(columnRef.getQualifier());
+ } else if (guessedRelations.size() > 1) {
+ throw new AmbiguousFieldException(columnRef.getCanonicalName());
+ }
+
+ String qualifier = guessedRelations.iterator().next().getCanonicalName();
+ String columnName = "";
+
+ if (columnNamePosition >= qualifierParts.length) { // if there is no column in qualifierParts
+ columnName = columnRef.getName();
} else {
- String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier());
- if (resolvedDatabaseName == null) {
- throw new NoSuchColumnException(columnRef.getQualifier());
+ // join a column name and its nested field names
+ columnName = qualifierParts[columnNamePosition];
+
+ // if qualifierParts include nested field names
+ if (qualifierParts.length > columnNamePosition) {
+ columnName += StringUtils.join(qualifierParts, "/", columnNamePosition + 1, qualifierParts.length);
}
- qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier());
- canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
+
+ // columnRef always has a leaf field name.
+ columnName += "/" + columnRef.getName();
}
- return new Pair<String, String>(qualifier, canonicalName);
+ return new Pair<String, String>(qualifier, columnName);
}
static Column ensureUniqueColumn(List<Column> candidates) throws VerifyException {
if (candidates.size() == 1) {
return candidates.get(0);
- } else if (candidates.size() > 2) {
+ } else if (candidates.size() > 1) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (Column column : candidates) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
index 19f39dd..0a665ab 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
@@ -48,12 +48,10 @@ public class ResolverByLegacy extends NameResolver {
private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
ColumnReferenceExpr columnRef) throws PlanningException {
final String qualifier;
- String canonicalName;
final String qualifiedName;
- Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef);
qualifier = normalized.getFirst();
- canonicalName = normalized.getSecond();
qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
Column found = resolveFromRelsWithinBlock(plan, block, columnRef);
@@ -84,7 +82,7 @@ public class ResolverByLegacy extends NameResolver {
&& currentNode.getType() != NodeType.TABLE_SUBQUERY) {
List<Column> candidates = TUtil.newList();
if (block.getNamedExprsManager().isAliased(qualifiedName)) {
- String alias = block.getNamedExprsManager().getAlias(canonicalName);
+ String alias = block.getNamedExprsManager().getAlias(qualifiedName);
found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY);
if (found != null) {
candidates.add(found);
@@ -101,7 +99,7 @@ public class ResolverByLegacy extends NameResolver {
static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
ColumnReferenceExpr columnRef)throws PlanningException {
- Column found = resolveFromAllRelsInBlock(block, columnRef);
+ Column found = lookupColumnFromAllRelsInBlock(block, columnRef.getName());
if (found != null) {
return found;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index dc6b8ef..cc54a22 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@ -726,7 +726,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
// find aggregation column
Set<Column> groupingColumns = TUtil.newHashSet(groupByNode.getGroupingColumns());
Set<String> aggrFunctionOutColumns = TUtil.newHashSet();
- for (Column column : groupByNode.getOutSchema().getColumns()) {
+ for (Column column : groupByNode.getOutSchema().getRootColumns()) {
if (!groupingColumns.contains(column)) {
aggrFunctionOutColumns.add(column.getQualifiedName());
}
@@ -842,7 +842,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
TableDesc table = scanNode.getTableDesc();
boolean hasQualifiedName = false;
if (table.hasPartition()) {
- for (Column c: table.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column c: table.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionColumns.add(c.getQualifiedName());
hasQualifiedName = c.hasQualifier();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index d8b6380..4b9fd48 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -224,7 +224,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod();
Schema paritionValuesSchema = new Schema();
- for (Column column : partitionDesc.getExpressionSchema().getColumns()) {
+ for (Column column : partitionDesc.getExpressionSchema().getRootColumns()) {
paritionValuesSchema.addColumn(column);
}
@@ -237,7 +237,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
// add qualifier to schema for qual
paritionValuesSchema.setQualifier(scanNode.getCanonicalName());
- for (Column column : paritionValuesSchema.getColumns()) {
+ for (Column column : paritionValuesSchema.getRootColumns()) {
for (EvalNode simpleExpr : conjunctiveForms) {
if (checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) {
indexablePredicateSet.add(simpleExpr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 763f938..3b83ded 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -146,7 +146,7 @@ public class PlannerUtil {
if (EvalTreeUtil.checkIfPartitionSelection(node, partSchema)) {
prefixPartitionWhere = true;
boolean isPrefix = true;
- for (Column c : partSchema.getColumns()) {
+ for (Column c : partSchema.getRootColumns()) {
String value = EvalTreeUtil.getPartitionValue(node, c.getSimpleName());
if (isPrefix && value == null)
isPrefix = false;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
index 22b3351..c5ef8ef 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
@@ -23,6 +23,8 @@ import org.apache.tajo.OverridableConf;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.util.PlannerUtil;
@@ -55,7 +57,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
/**
* It checks if an output schema of a projectable node and target's output data types are equivalent to each other.
*/
- private static void verifyProjectableOutputSchema(Projectable node) throws PlanningException {
+ private static void verifyProjectableOutputSchema(Context context, Projectable node) throws PlanningException {
Schema outputSchema = node.getOutSchema();
Schema targetSchema = PlannerUtil.targetToSchema(node.getTargets());
@@ -66,9 +68,15 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
}
for (int i = 0; i < outputSchema.size(); i++) {
- if (!outputSchema.getColumn(i).getDataType().equals(targetSchema.getColumn(i).getDataType())) {
+ Column outputColumn = outputSchema.getColumn(i);
+
+ if (outputColumn.getDataType().getType() == Type.RECORD) {
+ context.state.addVerification("Projecting RECORD fields is not supported yet.");
+ }
+
+ if (!outputColumn.getDataType().equals(targetSchema.getColumn(i).getDataType())) {
Column targetColumn = targetSchema.getColumn(i);
- Column insertColumn = outputSchema.getColumn(i);
+ Column insertColumn = outputColumn;
throw new PlanningException("ERROR: " +
insertColumn.getSimpleName() + " is of type " + insertColumn.getDataType().getType().name() +
", but target column '" + targetColumn.getSimpleName() + "' is of type " +
@@ -86,7 +94,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(state.state, node, target.getEvalTree());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(state, node);
return node;
}
@@ -108,7 +116,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
GroupbyNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitGroupBy(context, plan, block, node, stack);
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -130,7 +138,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(context.state, node, node.getJoinQual());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -192,7 +200,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
}
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -209,7 +217,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(context.state, node, node.getQual());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
index 275ab3a..8717bda 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
@@ -83,7 +83,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
super.visitProjection(context, stack, expr);
Set<String> names = TUtil.newHashSet();
-
+
for (NamedExpr namedExpr : expr.getNamedExprs()) {
if (namedExpr.hasAlias()) {
@@ -302,8 +302,8 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
return null;
}
if (table.hasPartition()) {
- int columnSize = table.getSchema().getColumns().size();
- columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size();
+ int columnSize = table.getSchema().getRootColumns().size();
+ columnSize += table.getPartitionMethod().getExpressionSchema().getRootColumns().size();
if (projectColumnNum < columnSize) {
context.state.addVerification("INSERT has smaller expressions than target columns");
} else if (projectColumnNum > columnSize) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
index 66b3667..5423fd7 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
@@ -84,7 +84,7 @@ public class MergeScanner implements Scanner {
tableStats.setNumBytes(numBytes);
tableStats.setNumBlocks(fragments.size());
- for(Column eachColumn: schema.getColumns()) {
+ for(Column eachColumn: schema.getRootColumns()) {
ColumnStats columnStats = new ColumnStats(eachColumn);
tableStats.addColumnStat(columnStats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 33db798..9d69423 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -35,7 +35,7 @@ public class RowStoreUtil {
public static int[] getTargetIds(Schema inSchema, Schema outSchema) {
int[] targetIds = new int[outSchema.size()];
int i = 0;
- for (Column target : outSchema.getColumns()) {
+ for (Column target : outSchema.getRootColumns()) {
targetIds[i] = inSchema.getColumnId(target.getQualifiedName());
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
index 0356b19..7af8247 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
@@ -61,6 +61,15 @@ public interface Scanner extends SchemaObject, Closeable {
/**
* It returns if the projection is executed in the underlying scanner layer.
*
+ * If TRUE, the upper layers (i.e., SeqScanExec) assume that next()
+ * will return a tuple which contains only projected fields. In other words,
+ * the field number of a retrieved tuple is equivalent tothe number of targets.
+ *
+ * If FALSE, the upper layers assume that next() will return a tuple which
+ * contains projected fields and non-projected fields, padded by NullDatum.
+ * In other words, the width of tuple is equivalent to the field number
+ * of the table schema.
+ *
* @return true if this scanner can project the given columns.
*/
boolean isProjectable();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
index ce963c8..b8438e9 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -409,9 +409,7 @@ public abstract class StorageManager {
Class<? extends Scanner> scannerClass = getScannerClass(meta.getStoreType());
scanner = newScannerInstance(scannerClass, conf, schema, meta, fragment);
- if (scanner.isProjectable()) {
- scanner.setTarget(target.toArray());
- }
+ scanner.setTarget(target.toArray());
return scanner;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
index 4836352..0e3441b 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
@@ -60,7 +60,6 @@ public abstract class AbstractHBaseAppender implements Appender {
protected boolean[] isRowKeyMappings;
protected boolean[] isColumnKeys;
protected boolean[] isColumnValues;
- protected int[] rowKeyFieldIndexes;
protected int[] rowkeyColumnIndexes;
protected char rowKeyDelimiter;
@@ -107,7 +106,6 @@ public abstract class AbstractHBaseAppender implements Appender {
isColumnKeys = columnMapping.getIsColumnKeys();
isColumnValues = columnMapping.getIsColumnValues();
rowKeyDelimiter = columnMapping.getRowKeyDelimiter();
- rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes();
this.columnNum = schema.size();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
index c3094fd..e66a707 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
@@ -77,7 +77,7 @@ public class ColumnMapping {
String[] columnMappingTokens = columnMapping.split(",");
- if (columnMappingTokens.length != schema.getColumns().size()) {
+ if (columnMappingTokens.length != schema.getRootColumns().size()) {
throw new IOException("The number of mapped HBase columns is great than the number of Tajo table columns");
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
index c05c5bb..53ff9dc 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
@@ -39,10 +39,14 @@ public class HBaseBinarySerializerDeserializer {
datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes));
break;
case INT8:
- if (bytes.length == 4) {
- datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes));
+ if (bytes == null) {
+ datum = NullDatum.get();
} else {
- datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes));
+ if (bytes.length == 4) {
+ datum = DatumFactory.createInt8(Bytes.toInt(bytes));
+ } else {
+ datum = bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes));
+ }
}
break;
case FLOAT4:
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
index ab56252..9eb1d86 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage.hbase;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -87,7 +88,13 @@ public class HBaseScanner implements Scanner {
private char rowKeyDelimiter;
public HBaseScanner (Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException {
- this.conf = (TajoConf)conf;
+ Preconditions.checkNotNull(conf);
+ Preconditions.checkNotNull(schema);
+ Preconditions.checkNotNull(meta);
+ Preconditions.checkNotNull(fragment);
+ Preconditions.checkArgument(conf instanceof TajoConf);
+
+ this.conf = (TajoConf) conf;
this.schema = schema;
this.meta = meta;
this.fragment = (HBaseFragment)fragment;
@@ -102,11 +109,10 @@ public class HBaseScanner implements Scanner {
tableStats.setNumBytes(0);
tableStats.setNumBlocks(1);
}
- if (schema != null) {
- for(Column eachColumn: schema.getColumns()) {
- ColumnStats columnStats = new ColumnStats(eachColumn);
- tableStats.addColumnStat(columnStats);
- }
+
+ for (Column eachColumn : schema.getRootColumns()) {
+ ColumnStats columnStats = new ColumnStats(eachColumn);
+ tableStats.addColumnStat(columnStats);
}
scanFetchSize = Integer.parseInt(
@@ -203,9 +209,9 @@ public class HBaseScanner implements Scanner {
}
Result result = scanResults[scanResultIndex++];
- Tuple resultTuple = new VTuple(schema.size());
+ Tuple resultTuple = new VTuple(targetIndexes.length);
for (int i = 0; i < targetIndexes.length; i++) {
- resultTuple.put(targetIndexes[i], getDatum(result, targetIndexes[i]));
+ resultTuple.put(i, getDatum(result, targetIndexes[i]));
}
numRows++;
return resultTuple;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
index a9e5bde..f74245e 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
@@ -355,7 +355,7 @@ public class HBaseStorageManager extends StorageManager {
Collection<String> columnFamilies = columnMapping.getColumnFamilyNames();
//If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column.
if (columnFamilies.isEmpty()) {
- for (Column eachColumn: schema.getColumns()) {
+ for (Column eachColumn: schema.getRootColumns()) {
columnFamilies.add(eachColumn.getSimpleName());
}
}
@@ -735,7 +735,7 @@ public class HBaseStorageManager extends StorageManager {
for (String property : CONNECTION_PROPERTIES) {
String thisValue = this.properties.get(property);
String thatValue = that.properties.get(property);
- //noinspection StringEquality
+ // noinspection StringEquality
if (thisValue == thatValue) {
continue;
}
@@ -1121,7 +1121,7 @@ public class HBaseStorageManager extends StorageManager {
try {
HTableDescriptor hTableDesc = parseHTableDescriptor(tableMeta, cNode.getTableSchema());
- LOG.info("Delete table cause query failed:" + hTableDesc.getName());
+ LOG.info("Delete table cause query failed:" + new String(hTableDesc.getName()));
hAdmin.disableTable(hTableDesc.getName());
hAdmin.deleteTable(hTableDesc.getName());
} finally {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
index c1047d9..5fc96f1 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -528,7 +528,7 @@ public class CSVFile {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
index 038f0f4..0726125 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
@@ -69,7 +69,7 @@ public abstract class FileScanner implements Scanner {
}
if (schema != null) {
- for(Column eachColumn: schema.getColumns()) {
+ for(Column eachColumn: schema.getRootColumns()) {
ColumnStats columnStats = new ColumnStats(eachColumn);
tableStats.addColumnStat(columnStats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
index 8d425b4..439bee0 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
@@ -732,7 +732,7 @@ public class FileStorageManager extends StorageManager {
//In the case of partitioned table, we should return same partition key data files.
int partitionDepth = 0;
if (tableDesc.hasPartition()) {
- partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size();
+ partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size();
}
List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
index 51594df..26083a5 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
@@ -80,8 +80,7 @@ public class AvroScanner extends FileScanner {
avroSchema = AvroUtil.getAvroSchema(meta, conf);
avroFields = avroSchema.getFields();
- DatumReader<GenericRecord> datumReader =
- new GenericDatumReader<GenericRecord>(avroSchema);
+ DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(avroSchema);
SeekableInput input = new FsInput(fragment.getPath(), conf);
dataFileReader = new DataFileReader<GenericRecord>(input, datumReader);
super.init();
@@ -175,13 +174,13 @@ public class AvroScanner extends FileScanner {
return null;
}
- Tuple tuple = new VTuple(schema.size());
+ Tuple tuple = new VTuple(projectionMap.length);
GenericRecord record = dataFileReader.next();
for (int i = 0; i < projectionMap.length; ++i) {
int columnIndex = projectionMap[i];
Object value = record.get(columnIndex);
if (value == null) {
- tuple.put(columnIndex, NullDatum.get());
+ tuple.put(i, NullDatum.get());
continue;
}
@@ -196,28 +195,28 @@ public class AvroScanner extends FileScanner {
TajoDataTypes.Type tajoType = dataType.getType();
switch (avroType) {
case NULL:
- tuple.put(columnIndex, NullDatum.get());
+ tuple.put(i, NullDatum.get());
break;
case BOOLEAN:
- tuple.put(columnIndex, DatumFactory.createBool((Boolean)value));
+ tuple.put(i, DatumFactory.createBool((Boolean)value));
break;
case INT:
- tuple.put(columnIndex, convertInt(value, tajoType));
+ tuple.put(i, convertInt(value, tajoType));
break;
case LONG:
- tuple.put(columnIndex, DatumFactory.createInt8((Long)value));
+ tuple.put(i, DatumFactory.createInt8((Long)value));
break;
case FLOAT:
- tuple.put(columnIndex, DatumFactory.createFloat4((Float)value));
+ tuple.put(i, DatumFactory.createFloat4((Float)value));
break;
case DOUBLE:
- tuple.put(columnIndex, DatumFactory.createFloat8((Double)value));
+ tuple.put(i, DatumFactory.createFloat8((Double)value));
break;
case BYTES:
- tuple.put(columnIndex, convertBytes(value, tajoType, dataType));
+ tuple.put(i, convertBytes(value, tajoType, dataType));
break;
case STRING:
- tuple.put(columnIndex, convertString(value, tajoType));
+ tuple.put(i, convertString(value, tajoType));
break;
case RECORD:
throw new RuntimeException("Avro RECORD not supported.");
@@ -228,7 +227,7 @@ public class AvroScanner extends FileScanner {
case UNION:
throw new RuntimeException("Avro UNION not supported.");
case FIXED:
- tuple.put(columnIndex, new BlobDatum(((GenericFixed)value).bytes()));
+ tuple.put(i, new BlobDatum(((GenericFixed)value).bytes()));
break;
default:
throw new RuntimeException("Unknown type.");