You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2015/05/11 23:36:27 UTC
[1/3] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
Repository: tajo
Updated Branches:
refs/heads/master ddd39213d -> 0d1bf41f3
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
index 204f607..0d1c94a 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
@@ -23,6 +23,7 @@ import io.netty.buffer.ByteBuf;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;
+import org.apache.tajo.catalog.*;
import org.apache.commons.net.util.Base64;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
@@ -35,26 +36,212 @@ import org.apache.tajo.datum.TextDatum;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.text.TextLineDeserializer;
import org.apache.tajo.storage.text.TextLineParsingError;
+import org.apache.tajo.util.StringUtils;
+import org.apache.tajo.util.TUtil;
import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
public class JsonLineDeserializer extends TextLineDeserializer {
private JSONParser parser;
- private Type[] types;
- private String[] columnNames;
+ // Full Path -> Type
+ private Map<String, Type> types;
+ private String [] projectedPaths;
- public JsonLineDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- super(schema, meta, targetColumnIndexes);
+ public JsonLineDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ super(schema, meta);
+
+ projectedPaths = new String[projected.length];
+ for (int i = 0; i < projected.length; i++) {
+ this.projectedPaths[i] = projected[i].getSimpleName();
+ }
}
@Override
public void init() {
- types = SchemaUtil.toTypes(schema);
- columnNames = SchemaUtil.toSimpleNames(schema);
+ types = TUtil.newHashMap();
+ for (Column column : schema.getAllColumns()) {
+ // Keep types which only belong to projected paths
+ // For example, assume that a projected path is 'name/first_name', where name is RECORD and first_name is TEXT.
+ // In this case, we should keep two types:
+ // * name - RECORD
+ // * name/first_name TEXT
+ for (String p :projectedPaths) {
+ if (p.startsWith(column.getSimpleName())) {
+ types.put(column.getSimpleName(), column.getDataType().getType());
+ }
+ }
+ }
parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR);
}
+ private static String makePath(String [] path, int depth) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i <= depth; i++) {
+ sb.append(path[i]);
+ if (i < depth) {
+ sb.append(NestedPathUtil.PATH_DELIMITER);
+ }
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ *
+ *
+ * @param object
+ * @param pathElements
+ * @param depth
+ * @param fieldIndex
+ * @param output
+ * @throws IOException
+ */
+ private void getValue(JSONObject object,
+ String fullPath,
+ String [] pathElements,
+ int depth,
+ int fieldIndex,
+ Tuple output) throws IOException {
+ String fieldName = pathElements[depth];
+
+ if (!object.containsKey(fieldName)) {
+ output.put(fieldIndex, NullDatum.get());
+ }
+
+ switch (types.get(fullPath)) {
+ case BOOLEAN:
+ String boolStr = object.getAsString(fieldName);
+ if (boolStr != null) {
+ output.put(fieldIndex, DatumFactory.createBool(boolStr.equals("true")));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case CHAR:
+ String charStr = object.getAsString(fieldName);
+ if (charStr != null) {
+ output.put(fieldIndex, DatumFactory.createChar(charStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case INT1:
+ case INT2:
+ Number int2Num = object.getAsNumber(fieldName);
+ if (int2Num != null) {
+ output.put(fieldIndex, DatumFactory.createInt2(int2Num.shortValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case INT4:
+ Number int4Num = object.getAsNumber(fieldName);
+ if (int4Num != null) {
+ output.put(fieldIndex, DatumFactory.createInt4(int4Num.intValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case INT8:
+ Number int8Num = object.getAsNumber(fieldName);
+ if (int8Num != null) {
+ output.put(fieldIndex, DatumFactory.createInt8(int8Num.longValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case FLOAT4:
+ Number float4Num = object.getAsNumber(fieldName);
+ if (float4Num != null) {
+ output.put(fieldIndex, DatumFactory.createFloat4(float4Num.floatValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case FLOAT8:
+ Number float8Num = object.getAsNumber(fieldName);
+ if (float8Num != null) {
+ output.put(fieldIndex, DatumFactory.createFloat8(float8Num.doubleValue()));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case TEXT:
+ String textStr = object.getAsString(fieldName);
+ if (textStr != null) {
+ output.put(fieldIndex, DatumFactory.createText(textStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case TIMESTAMP:
+ String timestampStr = object.getAsString(fieldName);
+ if (timestampStr != null) {
+ output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case TIME:
+ String timeStr = object.getAsString(fieldName);
+ if (timeStr != null) {
+ output.put(fieldIndex, DatumFactory.createTime(timeStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case DATE:
+ String dateStr = object.getAsString(fieldName);
+ if (dateStr != null) {
+ output.put(fieldIndex, DatumFactory.createDate(dateStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+ case BIT:
+ case BINARY:
+ case VARBINARY:
+ case BLOB: {
+ Object jsonObject = object.getAsString(fieldName);
+
+ if (jsonObject == null) {
+ output.put(fieldIndex, NullDatum.get());
+ break;
+ }
+
+ output.put(fieldIndex, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject)));
+ break;
+ }
+ case INET4:
+ String inetStr = object.getAsString(fieldName);
+ if (inetStr != null) {
+ output.put(fieldIndex, DatumFactory.createInet4(inetStr));
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+
+ case RECORD:
+ JSONObject nestedObject = (JSONObject) object.get(fieldName);
+ if (nestedObject != null) {
+ getValue(nestedObject, fullPath + "/" + pathElements[depth+1], pathElements, depth + 1, fieldIndex, output);
+ } else {
+ output.put(fieldIndex, NullDatum.get());
+ }
+ break;
+
+ case NULL_TYPE:
+ output.put(fieldIndex, NullDatum.get());
+ break;
+
+ default:
+ throw new NotImplementedException(types.get(fullPath).name() + " is not supported.");
+ }
+ }
+
@Override
public void deserialize(ByteBuf buf, Tuple output) throws IOException, TextLineParsingError {
byte[] line = new byte[buf.readableBytes()];
@@ -70,135 +257,9 @@ public class JsonLineDeserializer extends TextLineDeserializer {
throw new TextLineParsingError(new String(line, TextDatum.DEFAULT_CHARSET), ae);
}
- for (int i = 0; i < targetColumnIndexes.length; i++) {
- int actualIdx = targetColumnIndexes[i];
- String fieldName = columnNames[actualIdx];
-
- if (!object.containsKey(fieldName)) {
- output.put(actualIdx, NullDatum.get());
- continue;
- }
-
- switch (types[actualIdx]) {
- case BOOLEAN:
- String boolStr = object.getAsString(fieldName);
- if (boolStr != null) {
- output.put(actualIdx, DatumFactory.createBool(boolStr.equals("true")));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case CHAR:
- String charStr = object.getAsString(fieldName);
- if (charStr != null) {
- output.put(actualIdx, DatumFactory.createChar(charStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case INT1:
- case INT2:
- Number int2Num = object.getAsNumber(fieldName);
- if (int2Num != null) {
- output.put(actualIdx, DatumFactory.createInt2(int2Num.shortValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case INT4:
- Number int4Num = object.getAsNumber(fieldName);
- if (int4Num != null) {
- output.put(actualIdx, DatumFactory.createInt4(int4Num.intValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case INT8:
- Number int8Num = object.getAsNumber(fieldName);
- if (int8Num != null) {
- output.put(actualIdx, DatumFactory.createInt8(int8Num.longValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case FLOAT4:
- Number float4Num = object.getAsNumber(fieldName);
- if (float4Num != null) {
- output.put(actualIdx, DatumFactory.createFloat4(float4Num.floatValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case FLOAT8:
- Number float8Num = object.getAsNumber(fieldName);
- if (float8Num != null) {
- output.put(actualIdx, DatumFactory.createFloat8(float8Num.doubleValue()));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case TEXT:
- String textStr = object.getAsString(fieldName);
- if (textStr != null) {
- output.put(actualIdx, DatumFactory.createText(textStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case TIMESTAMP:
- String timestampStr = object.getAsString(fieldName);
- if (timestampStr != null) {
- output.put(actualIdx, DatumFactory.createTimestamp(timestampStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case TIME:
- String timeStr = object.getAsString(fieldName);
- if (timeStr != null) {
- output.put(actualIdx, DatumFactory.createTime(timeStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case DATE:
- String dateStr = object.getAsString(fieldName);
- if (dateStr != null) {
- output.put(actualIdx, DatumFactory.createDate(dateStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
- case BIT:
- case BINARY:
- case VARBINARY:
- case BLOB: {
- Object jsonObject = object.getAsString(fieldName);
-
- if (jsonObject == null) {
- output.put(actualIdx, NullDatum.get());
- break;
- }
-
- output.put(actualIdx, DatumFactory.createBlob(Base64.decodeBase64((String) jsonObject)));
- break;
- }
- case INET4:
- String inetStr = object.getAsString(fieldName);
- if (inetStr != null) {
- output.put(actualIdx, DatumFactory.createInet4(inetStr));
- } else {
- output.put(actualIdx, NullDatum.get());
- }
- break;
-
- case NULL_TYPE:
- output.put(actualIdx, NullDatum.get());
- break;
-
- default:
- throw new NotImplementedException(types[actualIdx].name() + " is not supported.");
- }
+ for (int i = 0; i < projectedPaths.length; i++) {
+ String [] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER);
+ getValue(object, paths[0], paths, 0, i, output);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
index 6db2c29..5f12d76 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage.json;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.storage.text.TextLineDeserializer;
@@ -26,8 +27,8 @@ import org.apache.tajo.storage.text.TextLineSerializer;
public class JsonLineSerDe extends TextLineSerDe {
@Override
- public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- return new JsonLineDeserializer(schema, meta, targetColumnIndexes);
+ public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ return new JsonLineDeserializer(schema, meta, projected);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
index d6faf2d..34e9661 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
@@ -35,8 +35,6 @@ import java.io.IOException;
import java.io.OutputStream;
public class JsonLineSerializer extends TextLineSerializer {
- private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
-
private Type [] types;
private String [] simpleNames;
private int columnNum;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
index a091eac..4c675a4 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoRecordConverter.java
@@ -77,10 +77,11 @@ public class TajoRecordConverter extends GroupConverter {
continue;
}
Type type = parquetSchema.getType(index);
+ final int writeIndex = i;
converters[index] = newConverter(column, type, new ParentValueContainer() {
@Override
void add(Object value) {
- TajoRecordConverter.this.set(projectionIndex, value);
+ TajoRecordConverter.this.set(writeIndex, value);
}
});
++index;
@@ -145,7 +146,7 @@ public class TajoRecordConverter extends GroupConverter {
*/
@Override
public void start() {
- currentTuple = new VTuple(tupleSize);
+ currentTuple = new VTuple(projectionMap.length);
}
/**
@@ -157,7 +158,7 @@ public class TajoRecordConverter extends GroupConverter {
final int projectionIndex = projectionMap[i];
Column column = tajoReadSchema.getColumn(projectionIndex);
if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE
- || currentTuple.get(projectionIndex) == null) {
+ || currentTuple.get(i) == null) {
set(projectionIndex, NullDatum.get());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
index 62e5ed9..af260b4 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
@@ -1635,7 +1635,7 @@ public class RCFile {
return null;
}
- Tuple tuple = new VTuple(schema.size());
+ Tuple tuple = new VTuple(targets.length);
getCurrentRow(tuple);
return tuple;
}
@@ -1705,16 +1705,16 @@ public class RCFile {
for (int j = 0; j < selectedColumns.length; ++j) {
SelectedColumn col = selectedColumns[j];
- int i = col.colIndex;
+ int actualColumnIdx = col.colIndex;
if (col.isNulled) {
- tuple.put(i, NullDatum.get());
+ tuple.put(j, NullDatum.get());
} else {
colAdvanceRow(j, col);
- Datum datum = serde.deserialize(schema.getColumn(i),
+ Datum datum = serde.deserialize(schema.getColumn(actualColumnIdx),
currentValue.loadedColumnsValueBuffer[j].getData(), col.rowReadIndex, col.prvLength, nullChars);
- tuple.put(i, datum);
+ tuple.put(j, datum);
col.rowReadIndex += col.prvLength;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
index 92a041c..af0973e 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
@@ -108,9 +108,9 @@ public class SequenceFileScanner extends FileScanner {
}
- fieldIsNull = new boolean[schema.getColumns().size()];
- fieldStart = new int[schema.getColumns().size()];
- fieldLength = new int[schema.getColumns().size()];
+ fieldIsNull = new boolean[schema.getRootColumns().size()];
+ fieldStart = new int[schema.getRootColumns().size()];
+ fieldLength = new int[schema.getRootColumns().size()];
prepareProjection(targets);
@@ -172,7 +172,7 @@ public class SequenceFileScanner extends FileScanner {
Text text = new Text();
reader.getCurrentValue(text);
cells = BytesUtils.splitPreserveAllTokens(text.getBytes(),
- delimiter, projectionMap, schema.getColumns().size());
+ delimiter, projectionMap, schema.getRootColumns().size());
totalBytes += (long)text.getBytes().length;
tuple = new LazyTuple(schema, cells, 0, nullChars, serde);
}
@@ -197,7 +197,7 @@ public class SequenceFileScanner extends FileScanner {
* So, tajo must make a tuple after parsing hive style BinarySerDe.
*/
private Tuple makeTuple(BytesWritable value) throws IOException{
- Tuple tuple = new VTuple(schema.getColumns().size());
+ Tuple tuple = new VTuple(schema.getRootColumns().size());
int start = 0;
int length = value.getLength();
@@ -213,7 +213,7 @@ public class SequenceFileScanner extends FileScanner {
int lastFieldByteEnd = start + 1;
// Go through all bytes in the byte[]
- for (int i = 0; i < schema.getColumns().size(); i++) {
+ for (int i = 0; i < schema.getRootColumns().size(); i++) {
fieldIsNull[i] = true;
if ((nullByte & (1 << (i % 8))) != 0) {
fieldIsNull[i] = false;
@@ -322,12 +322,12 @@ public class SequenceFileScanner extends FileScanner {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
public boolean isSelectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
index 03a0a26..0901c0b 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
@@ -19,6 +19,7 @@
package org.apache.tajo.storage.text;
import io.netty.buffer.ByteBuf;
+import org.apache.tajo.catalog.Column;
import io.netty.buffer.ByteBufProcessor;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -28,6 +29,7 @@ import org.apache.tajo.storage.FieldSerializerDeserializer;
import org.apache.tajo.storage.Tuple;
import java.io.IOException;
+import java.util.Arrays;
public class CSVLineDeserializer extends TextLineDeserializer {
private ByteBufProcessor processor;
@@ -35,8 +37,18 @@ public class CSVLineDeserializer extends TextLineDeserializer {
private ByteBuf nullChars;
private int delimiterCompensation;
- public CSVLineDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- super(schema, meta, targetColumnIndexes);
+ private int [] targetColumnIndexes;
+ private Column [] projected;
+
+ public CSVLineDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ super(schema, meta);
+
+ this.projected = projected;
+ targetColumnIndexes = new int[projected.length];
+ for (int i = 0; i < projected.length; i++) {
+ targetColumnIndexes[i] = schema.getColumnId(projected[i].getQualifiedName());
+ }
+ Arrays.sort(targetColumnIndexes);
}
@Override
@@ -66,7 +78,7 @@ public class CSVLineDeserializer extends TextLineDeserializer {
final int rowLength = lineBuf.readableBytes();
int start = 0, fieldLength = 0, end = 0;
- //Projection
+ // Projection
int currentTarget = 0;
int currentIndex = 0;
@@ -83,10 +95,10 @@ public class CSVLineDeserializer extends TextLineDeserializer {
lineBuf.setIndex(start, start + fieldLength);
try {
- Datum datum = fieldSerDer.deserialize(lineBuf, schema.getColumn(currentIndex), currentIndex, nullChars);
- output.put(currentIndex, datum);
+ Datum datum = fieldSerDer.deserialize(lineBuf, projected[currentTarget], currentIndex, nullChars);
+ output.put(currentTarget, datum);
} catch (Exception e) {
- output.put(currentIndex, NullDatum.get());
+ output.put(currentTarget, NullDatum.get());
}
currentTarget++;
@@ -103,7 +115,7 @@ public class CSVLineDeserializer extends TextLineDeserializer {
/* If a text row is less than table schema size, tuple should set to NullDatum */
if (projection.length > currentTarget) {
for (; currentTarget < projection.length; currentTarget++) {
- output.put(projection[currentTarget], NullDatum.get());
+ output.put(currentTarget, NullDatum.get());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
index 988d5d1..4ebdbe8 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
@@ -19,6 +19,7 @@
package org.apache.tajo.storage.text;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.storage.StorageConstants;
@@ -26,8 +27,8 @@ import org.apache.tajo.util.Bytes;
public class CSVLineSerDe extends TextLineSerDe {
@Override
- public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
- return new CSVLineDeserializer(schema, meta, targetColumnIndexes);
+ public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, Column [] projected) {
+ return new CSVLineDeserializer(schema, meta, projected);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
index 5e7bd94..55a2b96 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
@@ -275,7 +275,6 @@ public class DelimitedTextFile {
private final long endOffset;
/** The number of actual read records */
private int recordCount = 0;
- private int[] targetColumnIndexes;
private DelimitedLineReader reader;
private TextLineDeserializer deserializer;
@@ -321,13 +320,7 @@ public class DelimitedTextFile {
targets = schema.toArray();
}
- targetColumnIndexes = new int[targets.length];
- for (int i = 0; i < targets.length; i++) {
- targetColumnIndexes[i] = schema.getColumnId(targets[i].getQualifiedName());
- }
-
super.init();
- Arrays.sort(targetColumnIndexes);
if (LOG.isDebugEnabled()) {
LOG.debug("DelimitedTextFileScanner open:" + fragment.getPath() + "," + startOffset + "," + endOffset);
}
@@ -336,7 +329,7 @@ public class DelimitedTextFile {
reader.readLine(); // skip first line;
}
- deserializer = getLineSerde().createDeserializer(schema, meta, targetColumnIndexes);
+ deserializer = getLineSerde().createDeserializer(schema, meta, targets);
deserializer.init();
}
@@ -391,7 +384,7 @@ public class DelimitedTextFile {
return EmptyTuple.get();
}
- tuple = new VTuple(schema.size());
+ tuple = new VTuple(targets.length);
tuple.setOffset(offset);
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
index 89a7de9..f067cb3 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
@@ -31,12 +31,10 @@ import java.io.IOException;
public abstract class TextLineDeserializer {
protected final Schema schema;
protected final TableMeta meta;
- protected final int[] targetColumnIndexes;
- public TextLineDeserializer(Schema schema, TableMeta meta, int [] targetColumnIndexes) {
+ public TextLineDeserializer(Schema schema, TableMeta meta) {
this.schema = schema;
this.meta = meta;
- this.targetColumnIndexes = targetColumnIndexes;
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
index 1a53bb0..c09a83b 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
@@ -21,6 +21,7 @@ package org.apache.tajo.storage.text;
import io.netty.buffer.ByteBuf;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.datum.NullDatum;
@@ -36,7 +37,7 @@ public abstract class TextLineSerDe {
public TextLineSerDe() {
}
- public abstract TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int [] targetColumnIndexes);
+ public abstract TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, Column [] projected);
public abstract TextLineSerializer createSerializer(Schema schema, TableMeta meta);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index 561e2ef..322818d 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@ -175,7 +175,19 @@ public class TestMergeScanner {
Tuple tuple;
while ((tuple = scanner.next()) != null) {
totalCounts++;
- if (isProjectableStorage(meta.getStoreType())) {
+
+ if (storeType == StoreType.RAW) {
+ assertEquals(4, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ assertNotNull(tuple.get(2));
+ assertNotNull(tuple.get(3));
+ } else if (scanner.isProjectable()) {
+ assertEquals(2, tuple.size());
+ assertNotNull(tuple.get(0));
+ assertNotNull(tuple.get(1));
+ } else {
+ assertEquals(4, tuple.size());
assertNotNull(tuple.get(0));
assertNull(tuple.get(1));
assertNotNull(tuple.get(2));
@@ -189,14 +201,13 @@ public class TestMergeScanner {
private static boolean isProjectableStorage(StoreType type) {
switch (type) {
- case RCFILE:
- case PARQUET:
- case SEQUENCEFILE:
- case CSV:
- case AVRO:
- return true;
- default:
- return false;
+ case CSV:
+ case SEQUENCEFILE:
+ case RAW:
+ case ROWFILE:
+ return false;
+ default:
+ return true;
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 456ea00..a735307 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -297,15 +297,7 @@ public class TestStorages {
int tupleCnt = 0;
Tuple tuple;
while ((tuple = scanner.next()) != null) {
- if (storeType == StoreType.RCFILE
- || storeType == StoreType.CSV
- || storeType == StoreType.PARQUET
- || storeType == StoreType.SEQUENCEFILE
- || storeType == StoreType.AVRO) {
- assertTrue(tuple.get(0) == null);
- }
- assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
- assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
+ verifyProjectedFields(scanner.isProjectable(), tuple, tupleCnt);
tupleCnt++;
}
scanner.close();
@@ -313,6 +305,20 @@ public class TestStorages {
assertEquals(tupleNum, tupleCnt);
}
+ private void verifyProjectedFields(boolean projectable, Tuple tuple, int tupleCnt) {
+ if (projectable) {
+ assertTrue(tupleCnt + 2 == tuple.get(0).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(1).asFloat4());
+ } else {
+ // RAW and ROW always project all fields.
+ if (storeType != StoreType.RAW && storeType != StoreType.ROWFILE) {
+ assertTrue(tuple.get(0) == null);
+ }
+ assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
+ assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
+ }
+ }
+
@Test
public void testVariousTypes() throws IOException {
boolean handleProtobuf = storeType != StoreType.JSON;
@@ -956,7 +962,7 @@ public class TestStorages {
@Test
public void testLessThanSchemaSize() throws IOException {
/* RAW is internal storage. It must be same with schema size */
- if (storeType == StoreType.RAW || storeType == StoreType.AVRO){
+ if (storeType == StoreType.RAW || storeType == StoreType.AVRO || storeType == StoreType.PARQUET) {
return;
}
@@ -1008,7 +1014,12 @@ public class TestStorages {
Tuple tuple = scanner.next();
scanner.close();
- assertEquals(expect.get(1), tuple.get(1));
- assertEquals(NullDatum.get(), tuple.get(4));
+ if (scanner.isProjectable()) {
+ assertEquals(expect.get(1), tuple.get(0));
+ assertEquals(NullDatum.get(), tuple.get(1));
+ } else {
+ assertEquals(expect.get(1), tuple.get(1));
+ assertEquals(NullDatum.get(), tuple.get(4));
+ }
}
}
[3/3] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
Posted by hy...@apache.org.
TAJO-1359: Add nested field projector and language extension to project nested record. (hyunsik)
Closes #422
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0d1bf41f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0d1bf41f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0d1bf41f
Branch: refs/heads/master
Commit: 0d1bf41f3aff119af5d95270d735101a81d5721c
Parents: ddd3921
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon May 11 14:34:07 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon May 11 14:35:09 2015 -0700
----------------------------------------------------------------------
CHANGES | 7 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 12 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++
.../java/org/apache/tajo/catalog/Schema.java | 109 +++++-
.../org/apache/tajo/catalog/SchemaUtil.java | 33 +-
.../tajo/catalog/store/HiveCatalogStore.java | 4 +-
.../cli/tsql/commands/DescTableCommand.java | 4 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Tuple.java | 2 +-
.../java/org/apache/tajo/util/StringUtils.java | 70 ++++
.../main/java/org/apache/tajo/util/TUtil.java | 32 --
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +-
.../engine/planner/PhysicalPlannerImpl.java | 17 +-
.../tajo/engine/planner/global/DataChannel.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 6 +-
.../global/builder/DistinctGroupbyBuilder.java | 6 +-
.../DistinctGroupbyThirdAggregationExec.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 27 +-
.../tajo/engine/utils/TupleCacheScanner.java | 2 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 8 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../NonForwardQueryResultSystemScanner.java | 20 +-
.../resources/webapps/admin/catalogview.jsp | 5 +-
.../engine/function/TestFunctionLoader.java | 5 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 8 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 4 +-
.../tajo/engine/query/TestJoinBroadcast.java | 2 +-
.../engine/query/TestSelectNestedRecord.java | 71 ++++
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +-
.../rs/resources/TestQueryResultResource.java | 6 +-
.../TestSelectNestedRecord/sample1/table.json | 3 +
.../TestSelectNestedRecord/tweets/sample1.json | 4 +
.../TestSelectNestedRecord/sample1_ddl.sql | 7 +
.../TestSelectNestedRecord/sample2_ddl.sql | 19 ++
.../testNestedFieldAsGroupbyKey1.sql | 7 +
.../testNestedFieldAsJoinKey1.sql | 7 +
.../TestSelectNestedRecord/testSelect1.sql | 1 +
.../TestSelectNestedRecord/testSelect2.sql | 61 ++++
.../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++
.../testSelectWithParentheses2.sql | 2 +-
.../testNestedFieldAsGroupbyKey1.result | 6 +
.../testNestedFieldAsJoinKey1.result | 6 +
.../TestSelectNestedRecord/testSelect1.result | 5 +
.../TestSelectNestedRecord/testSelect2.result | 6 +
.../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +-
.../tajo/plan/LogicalPlanPreprocessor.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 32 +-
.../apache/tajo/plan/expr/RowConstantEval.java | 3 +-
.../tajo/plan/expr/WindowFunctionEval.java | 3 +-
.../function/python/PythonScriptEngine.java | 12 +-
.../GreedyHeuristicJoinOrderAlgorithm.java | 8 +-
.../apache/tajo/plan/joinorder/JoinEdge.java | 4 +-
.../apache/tajo/plan/joinorder/JoinGraph.java | 6 +-
.../tajo/plan/logical/DistinctGroupbyNode.java | 4 +-
.../apache/tajo/plan/logical/EvalExprNode.java | 3 +-
.../apache/tajo/plan/logical/GroupbyNode.java | 10 +-
.../tajo/plan/logical/ProjectionNode.java | 3 +-
.../apache/tajo/plan/logical/RelationNode.java | 12 +
.../tajo/plan/logical/ShuffleFileWriteNode.java | 3 +-
.../tajo/plan/logical/TruncateTableNode.java | 4 +-
.../apache/tajo/plan/logical/WindowAggNode.java | 7 +-
.../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++---
.../plan/nameresolver/ResolverByLegacy.java | 8 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 4 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 2 +-
.../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +-
.../plan/verifier/PreLogicalPlanVerifier.java | 6 +-
.../org/apache/tajo/storage/MergeScanner.java | 2 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 9 +
.../org/apache/tajo/storage/StorageManager.java | 4 +-
.../storage/hbase/AbstractHBaseAppender.java | 2 -
.../tajo/storage/hbase/ColumnMapping.java | 2 +-
.../HBaseBinarySerializerDeserializer.java | 10 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 22 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 6 +-
.../java/org/apache/tajo/storage/CSVFile.java | 2 +-
.../org/apache/tajo/storage/FileScanner.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 2 +-
.../apache/tajo/storage/avro/AvroScanner.java | 25 +-
.../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++--------
.../apache/tajo/storage/json/JsonLineSerDe.java | 5 +-
.../tajo/storage/json/JsonLineSerializer.java | 2 -
.../storage/parquet/TajoRecordConverter.java | 7 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 16 +-
.../tajo/storage/text/CSVLineDeserializer.java | 26 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 5 +-
.../tajo/storage/text/DelimitedTextFile.java | 11 +-
.../tajo/storage/text/TextLineDeserializer.java | 4 +-
.../apache/tajo/storage/text/TextLineSerDe.java | 3 +-
.../apache/tajo/storage/TestMergeScanner.java | 29 +-
.../org/apache/tajo/storage/TestStorages.java | 35 +-
96 files changed, 1279 insertions(+), 467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0aad306..84be66a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,8 +27,8 @@ Release 0.11.0 - unreleased
TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun,
Committed by hyunsik)
- TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates
- an important kind of information.
+ TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage()
+ eliminates an important kind of information.
(Contributed by Jongyoung Park, Committed by jihoon)
TAJO-1408: Make IntermediateEntryProto more compact.
@@ -277,6 +277,9 @@ Release 0.11.0 - unreleased
SUB TASKS
+ TAJO-1359: Add nested field projector and language extension to project
+ nested record. (hyunsik)
+
TAJO-1529: Implement json_extract_path_text(string, string) function.
(jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 12f36b1..8e5b657 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -171,6 +171,16 @@ public class CatalogUtil {
return openQuote && closeQuote;
}
+ /**
+ * True if a given name is a simple identifier, meaning is not a dot-chained name.
+ *
+ * @param columnOrTableName Column or Table name to be checked
+ * @return True if a given name is a simple identifier. Otherwise, it will return False.
+ */
+ public static boolean isSimpleIdentifier(String columnOrTableName) {
+ return columnOrTableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 1;
+ }
+
public static boolean isFQColumnName(String tableName) {
return tableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 3;
}
@@ -662,7 +672,7 @@ public class CatalogUtil {
if (types[i].getType() != Type.NULL_TYPE) {
Type candidate = TUtil.getFromNestedMap(OPERATION_CASTING_MAP, widest.getType(), types[i].getType());
if (candidate == null) {
- throw new InvalidOperationException("No matched operation for those types: " + TUtil.arrayToString
+ throw new InvalidOperationException("No matched operation for those types: " + StringUtils.join
(types));
}
widest = newSimpleDataType(candidate);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
new file mode 100644
index 0000000..58b4f26
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.catalog;
+
+import com.google.common.base.Preconditions;
+import org.apache.tajo.common.TajoDataTypes.Type;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Utility methods for nested field
+ */
+public class NestedPathUtil {
+ public static final String PATH_DELIMITER = "/";
+
+ public static final List<String> ROOT_PATH = Collections.unmodifiableList(new ArrayList<String>());
+
+ public static boolean isPath(String name) {
+ return name.indexOf(PATH_DELIMITER.charAt(0)) >= 0;
+ }
+
+ public static String makePath(String[] parts) {
+ return makePath(parts, 0);
+ }
+
+ public static String makePath(String[] parts, int startIndex) {
+ return makePath(parts, startIndex, parts.length);
+ }
+
+ /**
+ * Make a nested field path
+ *
+ * @param parts path parts
+ * @param startIndex startIndex
+ * @param depth Depth
+ * @return Path
+ */
+ public static String makePath(String[] parts, int startIndex, int depth) {
+ Preconditions.checkArgument(startIndex <= (parts.length - 1));
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = startIndex; i < depth; i++) {
+ sb.append(PATH_DELIMITER);
+ sb.append(parts[i].toString());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Lookup the actual column corresponding to a given path.
+ * We assume that a path starts with the slash '/' and it
+ * does not include the root field.
+ *
+ * @param nestedField Nested column
+ * @param path Path which starts with '/';
+ * @return Column corresponding to the path
+ */
+ public static Column lookupPath(Column nestedField, String path) {
+ Preconditions.checkArgument(path.charAt(0) == PATH_DELIMITER.charAt(0),
+ "A nested field path must start with slash '/'.");
+
+ // We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
+ // So, we should start the index from 1 instead of 0.
+ return lookupPath(nestedField, path.split(PATH_DELIMITER));
+ }
+
+ public static Column lookupPath(Column nestedField, String [] paths) {
+ // We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
+ // So, we should start the index from 1 instead of 0.
+ return lookupColumnInternal(nestedField, paths, 1);
+ }
+
+ private static Column lookupColumnInternal(Column currentColumn, String [] paths, int depth) {
+ Column found = null;
+
+ if (currentColumn.getDataType().getType() == Type.RECORD) {
+ found = currentColumn.typeDesc.nestedRecordSchema.getColumn(paths[depth]);
+ }
+
+ if (found != null) {
+ if (found.getDataType().getType() == Type.RECORD) {
+ return lookupColumnInternal(found, paths, depth + 1);
+ } else {
+ return found;
+ }
+ } else {
+ throw new NoSuchFieldError(makePath(paths));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 054cc2c..0e4b741 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -32,6 +32,7 @@ import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.json.GsonObject;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -135,7 +136,8 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @param qualifier The qualifier
*/
public void setQualifier(String qualifier) {
- List<Column> columns = getColumns();
+ // only change root fields, and must keep each nested field simple name
+ List<Column> columns = getRootColumns();
fields.clear();
fieldsByQualifiedName.clear();
@@ -180,14 +182,39 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @return The column matched to a given column name.
*/
public Column getColumn(String name) {
- String [] parts = name.split("\\.");
- // Some of the string can includes database name and table name and column name.
- // For example, it can be 'default.table1.id'.
- // Therefore, spilt string array length can be 3.
- if (parts.length >= 2) {
- return getColumnByQName(name);
+
+ if (NestedPathUtil.isPath(name)) {
+
+ // TODO - to be refactored
+ if (fieldsByQualifiedName.containsKey(name)) {
+ Column flattenColumn = fields.get(fieldsByQualifiedName.get(name));
+ if (flattenColumn != null) {
+ return flattenColumn;
+ }
+ }
+
+ String [] paths = name.split(NestedPathUtil.PATH_DELIMITER);
+ Column column = getColumn(paths[0]);
+ if (column == null) {
+ return null;
+ }
+ Column actualColumn = NestedPathUtil.lookupPath(column, paths);
+
+ Column columnPath = new Column(
+ column.getQualifiedName() + NestedPathUtil.makePath(paths, 1),
+ actualColumn.typeDesc);
+
+ return columnPath;
} else {
- return getColumnByName(name);
+ String[] parts = name.split("\\.");
+ // Some of the string can includes database name and table name and column name.
+ // For example, it can be 'default.table1.id'.
+ // Therefore, spilt string array length can be 3.
+ if (parts.length >= 2) {
+ return getColumnByQName(name);
+ } else {
+ return getColumnByName(name);
+ }
}
}
@@ -268,12 +295,46 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
return -1;
}
-
- public List<Column> getColumns() {
+
+ /**
+ * Get root columns, meaning all columns except for nested fields.
+ *
+ * @return A list of root columns
+ */
+ public List<Column> getRootColumns() {
return ImmutableList.copyOf(fields);
}
+ /**
+ * Get all columns, including all nested fields
+ *
+ * @return A list of all columns
+ */
+ public List<Column> getAllColumns() {
+ final List<Column> columnList = TUtil.newList();
+
+ SchemaUtil.visitSchema(this, new ColumnVisitor() {
+ @Override
+ public void visit(int depth, List<String> path, Column column) {
+ if (path.size() > 0) {
+ String parentPath = StringUtils.join(path, NestedPathUtil.PATH_DELIMITER);
+ String currentPath = parentPath + NestedPathUtil.PATH_DELIMITER + column.getSimpleName();
+ columnList.add(new Column(currentPath, column.getTypeDesc()));
+ } else {
+ columnList.add(column);
+ }
+ }
+ });
+
+ return columnList;
+ }
+
public boolean contains(String name) {
+ // TODO - It's a hack
+ if (NestedPathUtil.isPath(name)) {
+ return (getColumn(name) != null);
+ }
+
if (fieldsByQualifiedName.containsKey(name)) {
return true;
}
@@ -288,6 +349,11 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public boolean contains(Column column) {
+ // TODO - It's a hack
+ if (NestedPathUtil.isPath(column.getQualifiedName())) {
+ return (getColumn(column.getQualifiedName()) != null);
+ }
+
if (column.hasQualifier()) {
return fieldsByQualifiedName.containsKey(column.getQualifiedName());
} else {
@@ -314,7 +380,24 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public boolean containsAll(Collection<Column> columns) {
- return fields.containsAll(columns);
+ boolean containFlag = true;
+
+ for (Column c :columns) {
+ if (NestedPathUtil.isPath(c.getSimpleName())) {
+ if (contains(c.getQualifiedName())) {
+ containFlag &= true;
+ } else {
+ String[] paths = c.getQualifiedName().split("/");
+ boolean existRootPath = contains(paths[0]);
+ boolean existLeafPath = getColumn(c.getSimpleName()) != null;
+ containFlag &= existRootPath && existLeafPath;
+ }
+ } else {
+ containFlag &= fields.contains(c);
+ }
+ }
+
+ return containFlag;
}
public synchronized Schema addColumn(String name, TypeDesc typeDesc) {
@@ -351,7 +434,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public synchronized void addColumns(Schema schema) {
- for(Column column : schema.getColumns()) {
+ for(Column column : schema.getRootColumns()) {
addColumn(column);
}
}
@@ -396,7 +479,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
@Override
- public void visit(int depth, Column column) {
+ public void visit(int depth, List<String> path, Column column) {
if (column.getDataType().getType() == Type.RECORD) {
DataType.Builder updatedType = DataType.newBuilder(column.getDataType());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
index f2bb71c..c6b2f69 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
@@ -18,6 +18,10 @@
package org.apache.tajo.catalog;
+import org.apache.tajo.util.TUtil;
+
+import java.util.List;
+
import static org.apache.tajo.common.TajoDataTypes.DataType;
import static org.apache.tajo.common.TajoDataTypes.Type;
@@ -34,12 +38,12 @@ public class SchemaUtil {
static int tmpColumnSeq = 0;
public static Schema merge(Schema left, Schema right) {
Schema merged = new Schema();
- for(Column col : left.getColumns()) {
+ for(Column col : left.getRootColumns()) {
if (!merged.containsByQualifiedName(col.getQualifiedName())) {
merged.addColumn(col);
}
}
- for(Column col : right.getColumns()) {
+ for(Column col : right.getRootColumns()) {
if (merged.containsByQualifiedName(col.getQualifiedName())) {
merged.addColumn("?fake" + (tmpColumnSeq++), col.getDataType());
} else {
@@ -59,7 +63,7 @@ public class SchemaUtil {
*/
public static Schema getNaturalJoinColumns(Schema left, Schema right) {
Schema common = new Schema();
- for (Column outer : left.getColumns()) {
+ for (Column outer : left.getRootColumns()) {
if (!common.containsByName(outer.getSimpleName()) && right.containsByName(outer.getSimpleName())) {
common.addColumn(new Column(outer.getSimpleName(), outer.getDataType()));
}
@@ -113,7 +117,7 @@ public class SchemaUtil {
* Column visitor interface
*/
public static interface ColumnVisitor {
- public void visit(int depth, Column column);
+ public void visit(int depth, List<String> path, Column column);
}
/**
@@ -122,8 +126,8 @@ public class SchemaUtil {
* @param function
*/
public static void visitSchema(Schema schema, ColumnVisitor function) {
- for(Column col : schema.getColumns()) {
- visitInDepthFirstOrder(0, function, col);
+ for(Column col : schema.getRootColumns()) {
+ visitInDepthFirstOrder(0, NestedPathUtil.ROOT_PATH, function, col);
}
}
@@ -134,14 +138,21 @@ public class SchemaUtil {
* @param function Visitor
* @param column Current visiting column
*/
- private static void visitInDepthFirstOrder(int depth, ColumnVisitor function, Column column) {
+ private static void visitInDepthFirstOrder(int depth,
+ final List<String> path,
+ ColumnVisitor function,
+ Column column) {
+
if (column.getDataType().getType() == Type.RECORD) {
- for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getColumns()) {
- visitInDepthFirstOrder(depth + 1, function, nestedColumn);
+ for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getRootColumns()) {
+ List<String> newPath = TUtil.newList(path);
+ newPath.add(column.getQualifiedName());
+
+ visitInDepthFirstOrder(depth + 1, newPath, function, nestedColumn);
}
- function.visit(depth, column);
+ function.visit(depth, path, column);
} else {
- function.visit(depth, column);
+ function.visit(depth, path, column);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index 5b1a996..6acdbd1 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -453,7 +453,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
}
// set column information
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size());
for (Column eachField : columns) {
@@ -465,7 +465,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
// set partition keys
if (tableDesc.hasPartition() && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>();
- for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), ""));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index a3960e6..a5b53f2 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@ -27,7 +27,7 @@ import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.cli.tsql.TajoCli;
import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.List;
import java.util.Map;
@@ -126,7 +126,7 @@ public class DescTableCommand extends TajoShellCommand {
sb.append("type:").append(partition.getPartitionType().name()).append("\n");
sb.append("columns:").append(":");
- sb.append(TUtil.arrayToString(partition.getExpressionSchema().toArray()));
+ sb.append(StringUtils.join(partition.getExpressionSchema().toArray()));
}
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 385f99c..6e16095 100644
--- a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -36,7 +36,7 @@ public class RowStoreUtil {
public static int[] getTargetIds(Schema inSchema, Schema outSchema) {
int[] targetIds = new int[outSchema.size()];
int i = 0;
- for (Column target : outSchema.getColumns()) {
+ for (Column target : outSchema.getRootColumns()) {
targetIds[i] = inSchema.getColumnId(target.getQualifiedName());
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
index aec784f..fce10a3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
@@ -41,8 +41,8 @@ public interface Tuple extends Cloneable {
void put(Datum[] values);
+
Datum get(int fieldId);
-
void setOffset(long offset);
long getOffset();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index d035e4a..0a16072 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -371,4 +371,74 @@ public class StringUtils {
return resultArray;
}
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects Iterable objects
+ * @param delimiter Delimiter string
+ * @return A joined string
+ */
+ public static String join(Iterable objects, String delimiter) {
+ boolean first = true;
+ StringBuilder sb = new StringBuilder();
+ for(Object object : objects) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(delimiter);
+ }
+
+ sb.append(object.toString());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Concatenate all objects' string with the delimiter ", "
+ *
+ * @param objects Iterable objects
+ * @return A joined string
+ */
+ public static String join(Object[] objects) {
+ return join(objects, ", ", 0, objects.length);
+ }
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param startIndex the begin index to join
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter, int startIndex) {
+ return join(objects, delimiter, startIndex, objects.length);
+ }
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param startIndex the begin index to join
+ * @param length the number of columns to be joined
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter, int startIndex, int length) {
+ boolean first = true;
+ StringBuilder sb = new StringBuilder();
+ for(int i = startIndex; i + startIndex < length; i++) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(delimiter);
+ }
+
+ sb.append(objects[i].toString());
+ }
+
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
index 75f3e2a..2293ef5 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
@@ -240,38 +240,6 @@ public class TUtil {
}
}
- public static String collectionToString(Collection objects, String delimiter) {
- boolean first = true;
- StringBuilder sb = new StringBuilder();
- for(Object object : objects) {
- if (first) {
- first = false;
- } else {
- sb.append(delimiter);
- }
-
- sb.append(object.toString());
- }
-
- return sb.toString();
- }
-
- public static String arrayToString(Object [] objects) {
- boolean first = true;
- StringBuilder sb = new StringBuilder();
- for(Object object : objects) {
- if (first) {
- first = false;
- } else {
- sb.append(", ");
- }
-
- sb.append(object.toString());
- }
-
- return sb.toString();
- }
-
public static <T> T [] toArray(Collection<T> collection, Class<T> type) {
T array = (T) Array.newInstance(type, collection.size());
return collection.toArray((T[]) array);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 1a63470..3ab11bd 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -1307,7 +1307,7 @@ set_qualifier
;
column_reference
- : ((db_name = identifier DOT)? (tb_name=identifier DOT))? name=identifier
+ : identifier (DOT identifier)*
;
as_clause
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index 6d32fa5..7c99868 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1054,14 +1054,17 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
@Override
public ColumnReferenceExpr visitColumn_reference(SQLParser.Column_referenceContext ctx) {
- ColumnReferenceExpr column = new ColumnReferenceExpr(ctx.name.getText());
- if (checkIfExist(ctx.db_name)) {
- column.setQualifier(CatalogUtil.buildFQName(ctx.db_name.getText(), ctx.tb_name.getText()));
- } else if (ctx.tb_name != null) {
- column.setQualifier(ctx.tb_name.getText());
+ String columnReferenceName = ctx.getText();
+ // find the last dot (.) position to separate a name into both a qualifier and name
+ int lastDotIdx = columnReferenceName.lastIndexOf(".");
+
+ if (lastDotIdx > 0) { // if any qualifier is given
+ String qualifier = columnReferenceName.substring(0, lastDotIdx);
+ String name = columnReferenceName.substring(lastDotIdx + 1, columnReferenceName.length());
+ return new ColumnReferenceExpr(qualifier, name);
+ } else {
+ return new ColumnReferenceExpr(ctx.getText());
}
-
- return column;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index f132793..506b03e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -55,6 +55,7 @@ import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.IndexUtil;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -275,7 +276,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.",
context.getTaskId().toString(),
(left ? "Left" : "Right"),
- TUtil.arrayToString(lineage),
+ StringUtils.join(lineage),
FileUtil.humanReadableByteCount(volume, false),
(inMemoryInnerJoinFlag ? "" : "not ")));
return inMemoryInnerJoinFlag;
@@ -398,18 +399,18 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
larger = right;
LOG.info(String.format("[%s] Left relations %s (%s) is smaller than Right relations %s (%s).",
context.getTaskId().toString(),
- TUtil.arrayToString(leftLineage),
+ StringUtils.join(leftLineage),
FileUtil.humanReadableByteCount(leftSize, false),
- TUtil.arrayToString(rightLineage),
+ StringUtils.join(rightLineage),
FileUtil.humanReadableByteCount(rightSize, false)));
} else {
smaller = right;
larger = left;
LOG.info(String.format("[%s] Right relations %s (%s) is smaller than Left relations %s (%s).",
context.getTaskId().toString(),
- TUtil.arrayToString(rightLineage),
+ StringUtils.join(rightLineage),
FileUtil.humanReadableByteCount(rightSize, false),
- TUtil.arrayToString(leftLineage),
+ StringUtils.join(leftLineage),
FileUtil.humanReadableByteCount(leftSize, false)));
}
@@ -858,7 +859,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
} else if (storeTableNode.getType() == NodeType.CREATE_TABLE) {
int i = 0;
for (int j = 0; j < partitionKeyColumns.length; j++) {
- int id = storeTableNode.getOutSchema().getColumns().size() + j;
+ int id = storeTableNode.getOutSchema().getRootColumns().size() + j;
Column column = storeTableNode.getInSchema().getColumn(id);
sortSpecs[i++] = new SortSpec(column, true, false);
}
@@ -1002,7 +1003,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
sortNode.setInSchema(subOp.getSchema());
sortNode.setOutSchema(subOp.getSchema());
ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp);
- LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")");
+ LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")");
return new SortAggregateExec(ctx, groupbyNode, sortExec);
}
@@ -1043,7 +1044,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
sortNode.setInSchema(subOp.getSchema());
sortNode.setOutSchema(subOp.getSchema());
child = new ExternalSortExec(context, sortNode, subOp);
- LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")");
+ LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")");
}
return new WindowAggExec(context, windowAggNode, child);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
index 11548d3..ba1b0bf 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
@@ -23,7 +23,7 @@ import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
@@ -193,7 +193,7 @@ public class DataChannel {
sb.append(" (type=").append(shuffleType);
if (hasShuffleKeys()) {
sb.append(", key=");
- sb.append(TUtil.arrayToString(shuffleKeys));
+ sb.append(StringUtils.join(shuffleKeys));
sb.append(", num=").append(numOutputs);
}
sb.append(")");
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 36bdf21..ff1955e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -1146,11 +1146,11 @@ public class GlobalPlanner {
Column[] shuffleKeys = new Column[partitionMethod.getExpressionSchema().size()];
int i = 0, id = 0;
- for (Column column : partitionMethod.getExpressionSchema().getColumns()) {
+ for (Column column : partitionMethod.getExpressionSchema().getRootColumns()) {
if (node.getType() == NodeType.INSERT) {
id = tableSchema.getColumnId(column.getQualifiedName());
} else {
- id = tableSchema.getColumns().size() + i;
+ id = tableSchema.getRootColumns().size() + i;
}
shuffleKeys[i++] = projectedSchema.getColumn(id);
}
@@ -1493,7 +1493,7 @@ public class GlobalPlanner {
addedTableSubQueries.add(copy);
//Find a SubQueryNode which contains all columns in InputSchema matched with Target and OutputSchema's column
- if (copy.getInSchema().containsAll(copy.getOutSchema().getColumns())) {
+ if (copy.getInSchema().containsAll(copy.getOutSchema().getRootColumns())) {
for (Target eachTarget : copy.getTargets()) {
Set<Column> columns = EvalTreeUtil.findUniqueColumns(eachTarget.getEvalTree());
if (copy.getInSchema().containsAll(columns)) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
index b5e9104..13ed99b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
@@ -81,13 +81,13 @@ public class DistinctGroupbyBuilder {
// If there is not grouping column, we can't find column alias.
// Thus we should find the alias at Groupbynode output schema.
if (groupbyNode.getGroupingColumns().length == 0
- && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) {
+ && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) {
aggFunctions[i].setAlias(groupbyNode.getOutSchema().getColumn(i).getQualifiedName());
}
}
if (groupbyNode.getGroupingColumns().length == 0
- && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) {
+ && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) {
groupbyNode.setAggFunctions(aggFunctions);
}
@@ -672,7 +672,7 @@ public class DistinctGroupbyBuilder {
int index = 0;
for(GroupbyNode eachNode: secondStageDistinctNode.getSubPlans()) {
eachNode.setInSchema(firstStageDistinctNode.getOutSchema());
- for (Column column: eachNode.getOutSchema().getColumns()) {
+ for (Column column: eachNode.getOutSchema().getRootColumns()) {
if (secondStageInSchema.getColumn(column) == null) {
secondStageInSchema.addColumn(column);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
index e71976c..7c38d36 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
@@ -110,7 +110,7 @@ public class DistinctGroupbyThirdAggregationExec extends UnaryPhysicalExec {
}
int index = 0;
- for (Column eachOutputColumn: outSchema.getColumns()) {
+ for (Column eachOutputColumn: outSchema.getRootColumns()) {
// If column is avg aggregation function, outschema's column type is float
// but groupbyResultTupleIndex's column type is protobuf
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
index 247b373..7694b2b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
@@ -77,7 +77,7 @@ public class PhysicalPlanUtil {
//In the case of partitioned table, we should return same partition key data files.
int partitionDepth = 0;
if (tableDesc.hasPartition()) {
- partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size();
+ partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size();
}
List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
index ff9477f..3d95068 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
@@ -138,6 +138,8 @@ public class SeqScanExec extends ScanExec {
public void init() throws IOException {
Schema projected;
+ // in the case where projected column or expression are given
+ // the target can be an empty list.
if (plan.hasTargets()) {
projected = new Schema();
Set<Column> columnSet = new HashSet<Column>();
@@ -150,12 +152,15 @@ public class SeqScanExec extends ScanExec {
columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree()));
}
- for (Column column : inSchema.getColumns()) {
+ for (Column column : inSchema.getAllColumns()) {
if (columnSet.contains(column)) {
projected.addColumn(column);
}
}
+
} else {
+ // no any projected columns, meaning that all columns should be projected.
+ // TODO - this implicit rule makes code readability bad. So, we should remove it later
projected = outSchema;
}
@@ -163,7 +168,11 @@ public class SeqScanExec extends ScanExec {
super.init();
if (plan.hasQual()) {
- qual.bind(context.getEvalContext(), inSchema);
+ if (scanner.isProjectable()) {
+ qual.bind(context.getEvalContext(), projected);
+ } else {
+ qual.bind(context.getEvalContext(), inSchema);
+ }
}
}
@@ -175,7 +184,7 @@ public class SeqScanExec extends ScanExec {
}
private void initScanner(Schema projected) throws IOException {
- this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
+
TableMeta meta;
try {
meta = (TableMeta) plan.getTableDesc().getMeta().clone();
@@ -186,6 +195,7 @@ public class SeqScanExec extends ScanExec {
// set system default properties
PlannerUtil.applySystemDefaultToTableProperties(context.getQueryContext(), meta);
+ // Why we should check nullity? See https://issues.apache.org/jira/browse/TAJO-1422
if (fragments != null) {
if (fragments.length > 1) {
this.scanner = new MergeScanner(context.getConf(), plan.getPhysicalSchema(), meta,
@@ -198,6 +208,17 @@ public class SeqScanExec extends ScanExec {
plan.getPhysicalSchema(), fragments[0], projected);
}
scanner.init();
+
+ // See Scanner.isProjectable() method Depending on the result of isProjectable(),
+ // the width of retrieved tuple is changed.
+ //
+ // If TRUE, the retrieved tuple will contain only projected fields.
+ // If FALSE, the retrieved tuple will contain projected fields and NullDatum for non-projected fields.
+ if (scanner.isProjectable()) {
+ this.projector = new Projector(context, projected, outSchema, plan.getTargets());
+ } else {
+ this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
index ba25172..0fd2fbe 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
@@ -73,7 +73,7 @@ public class TupleCacheScanner implements Scanner {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
index 3a0a1c7..c01900a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
@@ -89,7 +89,7 @@ public class TupleUtil {
}
int i = 0;
- for (Column col : sortSchema.getColumns()) {
+ for (Column col : sortSchema.getRootColumns()) {
ColumnStats columnStat = statMap.get(col);
if (columnStat == null) {
continue;
@@ -121,7 +121,7 @@ public class TupleUtil {
statSet.put(stat.getColumn(), stat);
}
- for (Column col : target.getColumns()) {
+ for (Column col : target.getRootColumns()) {
Preconditions.checkState(statSet.containsKey(col),
"ERROR: Invalid Column Stats (column stats: " + colStats + ", there exists not target " + col);
}
@@ -134,7 +134,7 @@ public class TupleUtil {
// In outer join, empty table could be searched.
// As a result, min value and max value would be null.
// So, we should put NullDatum for this case.
- for (Column col : target.getColumns()) {
+ for (Column col : target.getRootColumns()) {
if (sortSpecs[sortSpecIndex].isAscending()) {
if (statSet.get(col).getMinValue() != null)
startTuple.put(i, statSet.get(col).getMinValue());
@@ -169,7 +169,7 @@ public class TupleUtil {
else
endTuple.put(i, DatumFactory.createNullDatum());
}
- if (target.getColumns().size() == sortSpecs.length) {
+ if (target.getRootColumns().size() == sortSpecs.length) {
// Not composite column sort
sortSpecIndex++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
index 804821b..9c0bd48 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
@@ -89,7 +89,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc
StringBuffer path = new StringBuffer();
int depth = 0;
if (tableDesc.hasPartition()) {
- for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
String partitionValue = EvalTreeUtil.getPartitionValue(scanNode.getQual(), c.getSimpleName());
if (partitionValue == null)
break;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
index 958c252..6c9b485 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
@@ -144,7 +144,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getTablespaces(Schema outSchema) {
List<TablespaceProto> tablespaces = masterContext.getCatalog().getAllTablespaces();
List<Tuple> tuples = new ArrayList<Tuple>(tablespaces.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablespaceProto tablespace: tablespaces) {
@@ -179,7 +179,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getDatabases(Schema outSchema) {
List<DatabaseProto> databases = masterContext.getCatalog().getAllDatabases();
List<Tuple> tuples = new ArrayList<Tuple>(databases.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (DatabaseProto database: databases) {
@@ -209,7 +209,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getTables(Schema outSchema) {
List<TableDescriptorProto> tables = masterContext.getCatalog().getAllTables();
List<Tuple> tuples = new ArrayList<Tuple>(tables.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableDescriptorProto table: tables) {
@@ -245,7 +245,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getColumns(Schema outSchema) {
List<ColumnProto> columnsList = masterContext.getCatalog().getAllColumns();
List<Tuple> tuples = new ArrayList<Tuple>(columnsList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
int columnId = 1, prevtid = -1, tid = 0;
@@ -293,7 +293,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getIndexes(Schema outSchema) {
List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes();
List<Tuple> tuples = new ArrayList<Tuple>(indexList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (IndexProto index: indexList) {
@@ -332,7 +332,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllTableOptions(Schema outSchema) {
List<TableOptionProto> optionList = masterContext.getCatalog().getAllTableOptions();
List<Tuple> tuples = new ArrayList<Tuple>(optionList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableOptionProto option: optionList) {
@@ -359,7 +359,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllTableStats(Schema outSchema) {
List<TableStatsProto> statList = masterContext.getCatalog().getAllTableStats();
List<Tuple> tuples = new ArrayList<Tuple>(statList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableStatsProto stat: statList) {
@@ -386,7 +386,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllPartitions(Schema outSchema) {
List<TablePartitionProto> partitionList = masterContext.getCatalog().getAllPartitions();
List<Tuple> tuples = new ArrayList<Tuple>(partitionList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablePartitionProto partition: partitionList) {
@@ -417,7 +417,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
}
private Tuple getQueryMasterTuple(Schema outSchema, Worker aWorker) {
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
WorkerResource aResource = aWorker.getResource();
@@ -463,7 +463,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
}
private Tuple getWorkerTuple(Schema outSchema, Worker aWorker) {
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
WorkerResource aResource = aWorker.getResource();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
index 1ff81a6..e014379 100644
--- a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
@@ -30,7 +30,6 @@
<%@ page import="java.util.Collection" %>
<%@ page import="java.util.List" %>
<%@ page import="java.util.Map" %>
-<%@ page import="org.apache.tajo.service.ServiceTracker" %>
<%
TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
CatalogService catalog = master.getCatalog();
@@ -143,7 +142,7 @@
<div style='margin-top:5px'>
<%
if(tableDesc != null) {
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
out.write("<table border='1' class='border_table'><tr><th>No</th><th>Column name</th><th>Type</th></tr>");
int columnIndex = 1;
for(Column eachColumn: columns) {
@@ -155,7 +154,7 @@
if (tableDesc.getPartitionMethod() != null) {
PartitionMethodDesc partition = tableDesc.getPartitionMethod();
- List<Column> partitionColumns = partition.getExpressionSchema().getColumns();
+ List<Column> partitionColumns = partition.getExpressionSchema().getRootColumns();
String partitionColumnStr = "";
String prefix = "";
for (Column eachColumn: partitionColumns) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
index 32b98dd..cf34c33 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
@@ -20,8 +20,7 @@ package org.apache.tajo.engine.function;
import com.google.common.collect.Lists;
import org.apache.tajo.catalog.FunctionDesc;
-import org.apache.tajo.function.FunctionSignature;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import org.junit.Test;
import java.io.IOException;
@@ -38,7 +37,7 @@ public class TestFunctionLoader {
public void testFindScalarFunctions() throws IOException {
List<FunctionDesc> collections = Lists.newArrayList(FunctionLoader.findScalarFunctions());
Collections.sort(collections);
- String functionList = TUtil.collectionToString(collections, "\n");
+ String functionList = StringUtils.join(collections, "\n");
String result = getResultText(TestFunctionLoader.class, "testFindScalarFunctions.result");
assertEquals(result.trim(), functionList.trim());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index af0aa6a..dfac53d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -871,7 +871,7 @@ public class TestLogicalPlanner {
assertEquals(NodeType.EXPRS, root.getChild().getType());
Schema out = root.getOutSchema();
- Iterator<Column> it = out.getColumns().iterator();
+ Iterator<Column> it = out.getRootColumns().iterator();
Column col = it.next();
assertEquals("res1", col.getSimpleName());
col = it.next();
@@ -920,7 +920,7 @@ public class TestLogicalPlanner {
testJsonSerDerObject(root);
Schema finalSchema = root.getOutSchema();
- Iterator<Column> it = finalSchema.getColumns().iterator();
+ Iterator<Column> it = finalSchema.getRootColumns().iterator();
Column col = it.next();
assertEquals("deptname", col.getSimpleName());
col = it.next();
@@ -931,7 +931,7 @@ public class TestLogicalPlanner {
root = (LogicalRootNode) plan;
finalSchema = root.getOutSchema();
- it = finalSchema.getColumns().iterator();
+ it = finalSchema.getRootColumns().iterator();
col = it.next();
assertEquals("id", col.getSimpleName());
col = it.next();
@@ -948,7 +948,7 @@ public class TestLogicalPlanner {
testJsonSerDerObject(root);
Schema finalSchema = root.getOutSchema();
- Iterator<Column> it = finalSchema.getColumns().iterator();
+ Iterator<Column> it = finalSchema.getRootColumns().iterator();
Column col = it.next();
assertEquals("id", col.getSimpleName());
col = it.next();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index f79f703..727b1a2 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -68,7 +68,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertTrue(desc.getSchema().contains("default.testctaswithouttabledefinition.col1"));
PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
- assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
+ assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration());
Path path = new Path(desc.getPath());
@@ -111,7 +111,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
- assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
+ assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(cluster.getConfiguration());
Path path = new Path(desc.getPath());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
index 48aea26..a1eceea 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
@@ -707,7 +707,7 @@ public class TestJoinBroadcast extends QueryTestCaseBase {
"select a.o_orderdate, a.o_orderstatus, a.o_orderkey_mod, a.o_totalprice " +
"from " + tableName +
" a join "+ tableName + " b on a.o_orderkey = b.o_orderkey " +
- "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and o_orderkey_mod = 1 " +
+ "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and a.o_orderkey_mod = 1 " +
" order by a.o_orderkey"
);
String resultSetData = resultSetToString(res);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
new file mode 100644
index 0000000..9f8a5fd
--- /dev/null
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.util.TUtil;
+import org.junit.Test;
+
+import java.sql.ResultSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestSelectNestedRecord extends QueryTestCaseBase {
+
+ @Test
+ public final void testSelect1() throws Exception {
+ List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample1");
+ assertEquals(TUtil.newList("sample1"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testSelect2() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testNestedFieldAsGroupbyKey1() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testNestedFieldAsJoinKey1() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
index 0dbb8e5..c8c24cd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
@@ -262,7 +262,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
TableDesc tableDesc = client.getTableDesc(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
assertNotNull(tableDesc);
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
while (rs.next()) {
assertEquals(tableName, rs.getString("TABLE_NAME"));
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
index 6fc4ea1..1b23966 100644
--- a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
+++ b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
@@ -140,7 +140,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
@@ -174,7 +174,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
@@ -236,7 +236,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
new file mode 100644
index 0000000..db3ad6c
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
@@ -0,0 +1,3 @@
+{ "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}}
+{ "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}}
+{ "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
new file mode 100644
index 0000000..78a9071
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
@@ -0,0 +1,4 @@
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Mon Sep 24 03:35:21 +0000 2012","id_str":"250075927172759552","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Aggressive Ponytail #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":1,"in_reply_to_status_id_str":null,"id":250075927172759552,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDEEF6","profile_sidebar_border_color":"C0DEED","profile_background_tile":false,"name":"Sean Cummings","profile_image_url":"http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","created_at":"Mon Apr 26 06:01:55 +0000 2010","location":"LA, CA","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"137238150","entities":{"url":{"urls":[{"expanded_url":null,"url":"","i
ndices":[0,0]}]},"description":{"urls":[]}},"default_profile":true,"contributors_enabled":false,"favourites_count":0,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","utc_offset":-28800,"id":137238150,"profile_use_background_image":true,"listed_count":2,"profile_text_color":"333333","lang":"en","followers_count":70,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_background_color":"C0DEED","verified":false,"geo_enabled":true,"time_zone":"Pacific Time (US & Canada)","description":"Born 330 Live 310","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme1/bg.png","statuses_count":579,"friends_count":110,"following":null,"show_all_inline_media":false,"screen_name":"sean_cummings"},"in_reply_to_screen_name":null,"source":"<a>Twitter for Mac<\/a>","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:40:54 +0000 2012","id_str":"249292149810667520","entities":{"urls":[],"hashtags":[{"text":"FreeBandNames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Thee Namaste Nerdz. #FreeBandNames","metadata":{"iso_language_code":"pl","result_type":"recent"},"retweet_count":2,"in_reply_to_status_id_str":null,"id":249292149810667520,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDFFCC","profile_sidebar_border_color":"BDDCAD","profile_background_tile":true,"name":"Chaz Martenstein","profile_image_url":"http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","created_at":"Tue Apr 07 19:05:07 +0000 2009","location":"Durham, NC","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"29516238","entities":{"url":{"urls":[{"expanded_url":null,"url":"http://bu
llcityrecords.com/wnng/","indices":[0,32]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":8,"url":"http://bullcityrecords.com/wnng/","profile_image_url_https":"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","utc_offset":-18000,"id":29516238,"profile_use_background_image":true,"listed_count":118,"profile_text_color":"333333","lang":"en","followers_count":2052,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp","profile_background_color":"9AE4E8","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp","statuses_c
ount":7579,"friends_count":348,"following":null,"show_all_inline_media":true,"screen_name":"bullcityrecords"},"in_reply_to_screen_name":null,"source":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:30:20 +0000 2012","id_str":"249289491129438208","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[29,43]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Mexican Heaven, Mexican Hell #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":3,"in_reply_to_status_id_str":null,"id":249289491129438208,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"99CC33","profile_sidebar_border_color":"829D5E","profile_background_tile":false,"name":"Thomas John Wakeman","profile_image_url":"http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","created_at":"Tue Sep 01 21:21:35 +0000 2009","location":"Kingston New York","follow_request_sent":null,"profile_link_color":"D02B55","is_translator":false,"id_str":"70789458","entities":{"url":{"urls":[{"expanded_url":n
ull,"url":"","indices":[0,0]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":19,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","utc_offset":-18000,"id":70789458,"profile_use_background_image":true,"listed_count":1,"profile_text_color":"3E4415","lang":"en","followers_count":63,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_background_color":"352726","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme5/bg.gif","statuses_count":1048,"friends_count":63,"following":null,"show_all_inline_media":false,"screen_name":"MonkiesFist"},"in_reply_to_screen_name":null,"sour
ce":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 22:51:18 +0000 2012","id_str":"249279667666817024","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"The Foolish Mortals #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":4,"in_reply_to_status_id_str":null,"id":249279667666817024,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"BFAC83","profile_sidebar_border_color":"615A44","profile_background_tile":true,"name":"Marty Elmer","profile_image_url":"http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","created_at":"Mon May 04 00:05:00 +0000 2009","location":"Wisconsin, USA","follow_request_sent":null,"profile_link_color":"3B2A26","is_translator":false,"id_str":"37539828","entities":{"url":{"urls":[{"expanded_url":null,"url":"ht
tp://www.omnitarian.me","indices":[0,24]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":647,"url":"http://www.omnitarian.me","profile_image_url_https":"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","utc_offset":-21600,"id":37539828,"profile_use_background_image":true,"listed_count":52,"profile_text_color":"000000","lang":"en","followers_count":608,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png","profile_background_color":"EEE3C4","verified":false,"geo_enabled":false,"time_zone":"Central Time (US & Canada)","description":"Cartoonist, Illustrator, and T-Shirt connoisseur","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png","statuses_count":3575,"friends_count":249,"following":null,"show_all_inline_media":true,"scree
n_name":"Omnitarian"},"in_reply_to_screen_name":null,"source":"<a>Twitter for iPhone<\/a>","in_reply_to_status_id":null}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
new file mode 100644
index 0000000..9ba5f8c
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
@@ -0,0 +1,7 @@
+CREATE EXTERNAL TABLE ${0} (
+ title TEXT,
+ name RECORD (
+ first_name TEXT,
+ last_name TEXT
+ )
+) USING JSON LOCATION ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
new file mode 100644
index 0000000..9537c3e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
@@ -0,0 +1,19 @@
+CREATE EXTERNAL TABLE ${0} (
+ glossary RECORD (
+ title TEXT,
+ "GlossDiv" RECORD (
+ "GlossEntry" RECORD (
+ "ID" TEXT,
+ "SortAs" TEXT,
+ "GlossTerm" TEXT,
+ "Acronym" TEXT,
+ "Abbrev" TEXT,
+ "GlossDef" RECORD (
+ para TEXT
+ ),
+
+ "GlossSee" TEXT
+ )
+ )
+ )
+) USING JSON LOCATION ${path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
new file mode 100644
index 0000000..057ba05
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
@@ -0,0 +1,7 @@
+SELECT
+ user.name,
+ sum(retweet_count) as total_retweet
+FROM
+ tweets
+GROUP BY
+ user.name;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
new file mode 100644
index 0000000..336840e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
@@ -0,0 +1,7 @@
+SELECT
+ t1.user.id,
+ t1.user.name,
+ t2.user.id,
+ t2.user.name
+FROM
+ tweets t1 join tweets t2 ON t1.user.id = t2.user.id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
new file mode 100644
index 0000000..b099e77
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
@@ -0,0 +1 @@
+SELECT title, (name.first_name || ' ' || name.last_name) as full_name FROM sample1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
new file mode 100644
index 0000000..9993a4e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
@@ -0,0 +1,61 @@
+SELECT
+ coordinates,
+ favorited,
+ truncated,
+ created_at,
+ id_str,
+ in_reply_to_user_id_str,
+ contributors,
+ "text",
+ metadata.iso_language_code,
+ metadata.result_type,
+ retweet_count,
+ in_reply_to_status_id_str,
+ id,
+ geo,
+ retweeted,
+ in_reply_to_user_id,
+ place,
+ user.profile_sidebar_fill_color,
+ user.profile_sidebar_border_color,
+ user.profile_background_tile,
+ user.name,
+ user.profile_image_url,
+ user.created_at,
+ user.location,
+ user.follow_request_sent,
+ user.profile_link_color,
+ user.is_translator,
+ user.id_str,
+ user.default_profile,
+ user.contributors_enabled,
+ user.favourites_count,
+ user.url,
+ user.profile_image_url_https,
+ user.utc_offset,
+ user.id,
+ user.profile_use_background_image,
+ user.listed_count,
+ user.profile_text_color,
+ user.lang,
+ user.followers_count,
+ user.protected,
+ user.notifications,
+ user.profile_background_image_url_https,
+ user.profile_background_color,
+ user.verified,
+ user.geo_enabled,
+ user.time_zone,
+ user.description,
+ user.default_profile_image,
+ user.profile_background_image_url,
+ user.statuses_count,
+ user.friends_count,
+ user.following,
+ user.show_all_inline_media,
+ user.screen_name,
+ in_reply_to_screen_name,
+ source,
+ in_reply_to_status_id
+FROM
+ tweets;
\ No newline at end of file
[2/3] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
Posted by hy...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
new file mode 100644
index 0000000..e750095
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
@@ -0,0 +1,74 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS ${0} (
+ coordinates TEXT,
+ favorited BOOL,
+ truncated BOOL,
+ created_at TIMESTAMP,
+ id_str TEXT,
+ /*entrities RECORD ( -- when we support array, we should remove this comment.
+ urls ARRAY<TEXT>
+ )*/
+ in_reply_to_user_id_str TEXT,
+ contributors TEXT,
+ text TEXT,
+ metadata RECORD (
+ iso_language_code TEXT,
+ result_type TEXT
+ ),
+ retweet_count INTEGER,
+ in_reply_to_status_id_str TEXT,
+ id TEXT,
+ geo TEXT,
+ retweeted BOOL,
+ in_reply_to_user_id TEXT,
+ place TEXT,
+ user RECORD (
+ profile_sidebar_fill_color TEXT,
+ profile_sidebar_border_color TEXT,
+ profile_background_tile TEXT,
+ name TEXT,
+ profile_image_url TEXT,
+ created_at TIMESTAMP,
+ location TEXT,
+ follow_request_sent TEXT,
+ profile_link_color TEXT,
+ is_translator BOOL,
+ id_str TEXT,
+ /* -- when we support array, we should fill the following comments.
+ entities RECORD (
+ url RECORD (
+ ),
+ description RECORD (
+ )
+ ), */
+ default_profile BOOL,
+ contributors_enabled BOOL,
+ favourites_count INTEGER,
+ url TEXT,
+ profile_image_url_https TEXT,
+ utc_offset INTEGER,
+ id BIGINT,
+ profile_use_background_image BOOL,
+ listed_count INTEGER,
+ profile_text_color TEXT,
+ lang TEXT,
+ followers_count INTEGER,
+ protected BOOL,
+ notifications TEXT,
+ profile_background_image_url_https TEXT,
+ profile_background_color TEXT,
+ verified TEXT,
+ geo_enabled TEXT,
+ time_zone TEXT,
+ description TEXT,
+ default_profile_image TEXT,
+ profile_background_image_url TEXT,
+ statuses_count INTEGER,
+ friends_count INTEGER,
+ following TEXT,
+ show_all_inline_media BOOL,
+ screen_name TEXT
+ ),
+ in_reply_to_screen_name TEXT,
+ source TEXT,
+ in_reply_to_status_id TEXT
+) USING JSON LOCATION ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
index 1760c99..e707a8c 100644
--- a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
@@ -1 +1 @@
-(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n_nationkey = 1);
+(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n1.n_nationkey = 1);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
new file mode 100644
index 0000000..debf06e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
@@ -0,0 +1,6 @@
+user/name,total_retweet
+-------------------------------
+Chaz Martenstein,2
+Thomas John Wakeman,3
+Sean Cummings,1
+Marty Elmer,4
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
new file mode 100644
index 0000000..1c57dc2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
@@ -0,0 +1,6 @@
+user/id,user/name,user/id,user/name
+-------------------------------
+137238150,Sean Cummings,137238150,Sean Cummings
+29516238,Chaz Martenstein,29516238,Chaz Martenstein
+70789458,Thomas John Wakeman,70789458,Thomas John Wakeman
+37539828,Marty Elmer,37539828,Marty Elmer
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
new file mode 100644
index 0000000..f96fcc5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
@@ -0,0 +1,5 @@
+title,full_name
+-------------------------------
+Hand of the King,Eddard Stark
+Assassin,Arya Stark
+Dancing Master,Syrio Forel
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
new file mode 100644
index 0000000..90eadbd
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
@@ -0,0 +1,6 @@
+coordinates,favorited,truncated,created_at,id_str,in_reply_to_user_id_str,contributors,text,metadata/iso_language_code,metadata/result_type,retweet_count,in_reply_to_status_id_str,id,geo,retweeted,in_reply_to_user_id,place,user/profile_sidebar_fill_color,user/profile_sidebar_border_color,user/profile_background_tile,user/name,user/profile_image_url,user/created_at,user/location,user/follow_request_sent,user/profile_link_color,user/is_translator,user/id_str,user/default_profile,user/contributors_enabled,user/favourites_count,user/url,user/profile_image_url_https,user/utc_offset,user/id,user/profile_use_background_image,user/listed_count,user/profile_text_color,user/lang,user/followers_count,user/protected,user/notifications,user/profile_background_image_url_https,user/profile_background_color,user/verified,user/geo_enabled,user/time_zone,user/description,user/default_profile_image,user/profile_background_image_url,user/statuses_count,user/friends_count,user/following,user/show_all_in
line_media,user/screen_name,in_reply_to_screen_name,source,in_reply_to_status_id
+-------------------------------
+null,false,false,2012-09-24 03:35:21,250075927172759552,null,null,Aggressive Ponytail #freebandnames,en,recent,1,null,250075927172759552,null,false,null,null,DDEEF6,C0DEED,false,Sean Cummings,http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,2010-04-26 06:01:55,LA, CA,null,0084B4,false,137238150,true,false,0,null,https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,-28800,137238150,true,2,333333,en,70,false,null,https://si0.twimg.com/images/themes/theme1/bg.png,C0DEED,false,true,Pacific Time (US & Canada),Born 330 Live 310,false,http://a0.twimg.com/images/themes/theme1/bg.png,579,110,null,false,sean_cummings,null,<a>Twitter for Mac</a>,null
+null,false,false,2012-09-21 23:40:54,249292149810667520,null,null,Thee Namaste Nerdz. #FreeBandNames,pl,recent,2,null,249292149810667520,null,false,null,null,DDFFCC,BDDCAD,true,Chaz Martenstein,http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,2009-04-07 19:05:07,Durham, NC,null,0084B4,false,29516238,false,false,8,http://bullcityrecords.com/wnng/,https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,-18000,29516238,true,118,333333,en,2052,false,null,https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp,9AE4E8,false,false,Eastern Time (US & Canada),You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.,false,http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp,7579,348,null,true,bullcityrecords,null,web,null
+null,false,false,2012-09-21 23:30:20,249289491129438208,null,null,Mexican Heaven, Mexican Hell #freebandnames,en,recent,3,null,249289491129438208,null,false,null,null,99CC33,829D5E,false,Thomas John Wakeman,http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,2009-09-01 21:21:35,Kingston New York,null,D02B55,false,70789458,false,false,19,null,https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,-18000,70789458,true,1,3E4415,en,63,false,null,https://si0.twimg.com/images/themes/theme5/bg.gif,352726,false,false,Eastern Time (US & Canada),Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.,false,http://a0.twimg.com/images/themes/theme5/bg.gif,1048,63,null,false,MonkiesFist,null,web,null
+null,false,false,2012-09-21 22:51:18,249279667666817024,null,null,The Foolish Mortals #freebandnames,en,recent,4,null,249279667666817024,null,false,null,null,BFAC83,615A44,true,Marty Elmer,http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,2009-05-04 00:05:00,Wisconsin, USA,null,3B2A26,false,37539828,false,false,647,http://www.omnitarian.me,https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,-21600,37539828,true,52,000000,en,608,false,null,https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png,EEE3C4,false,false,Central Time (US & Canada),Cartoonist, Illustrator, and T-Shirt connoisseur,false,http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png,3575,249,null,true,Omnitarian,null,<a>Twitter for iPhone</a>,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
index 2368082..d4ef55e 100644
--- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
+++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
@@ -513,7 +513,7 @@ public class TajoDatabaseMetaData implements DatabaseMetaData {
CatalogUtil.buildFQName(databaseName, table));
int pos = 0;
- for (Column column: tableDesc.getLogicalSchema().getColumns()) {
+ for (Column column: tableDesc.getLogicalSchema().getRootColumns()) {
if (column.getSimpleName().matches(regcolumnNamePattern)) {
MetaDataTuple tuple = new MetaDataTuple(22);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
index 6262995..fbad76e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
@@ -114,7 +114,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
Schema schema = relationOp.getLogicalSchema();
Column[] resolvedColumns = new Column[schema.size()];
- return schema.getColumns().toArray(resolvedColumns);
+ return schema.getRootColumns().toArray(resolvedColumns);
} else { // if a column reference is not qualified
// columns of every relation should be resolved.
Iterator<RelationNode> iterator = block.getRelations().iterator();
@@ -124,7 +124,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
while (iterator.hasNext()) {
relationOp = iterator.next();
schema = relationOp.getLogicalSchema();
- resolvedColumns.addAll(schema.getColumns());
+ resolvedColumns.addAll(schema.getRootColumns());
}
if (resolvedColumns.size() == 0) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index e0b4f7e..21270e9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -52,6 +52,7 @@ import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -527,13 +528,24 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
} else if (projectable instanceof RelationNode) {
RelationNode relationNode = (RelationNode) projectable;
+ prohibitNestedRecordProjection((Projectable) relationNode);
verifyIfTargetsCanBeEvaluated(relationNode.getLogicalSchema(), (Projectable) relationNode);
} else {
+ prohibitNestedRecordProjection(projectable);
verifyIfTargetsCanBeEvaluated(projectable.getInSchema(), projectable);
}
}
+ public static void prohibitNestedRecordProjection(Projectable projectable)
+ throws PlanningException {
+ for (Target t : projectable.getTargets()) {
+ if (t.getEvalTree().getValueType().getType() == TajoDataTypes.Type.RECORD) {
+ throw new PlanningException("Projecting RECORD fields is not supported yet: " + t);
+ }
+ }
+ }
+
public static void verifyIfEvalNodesCanBeEvaluated(Projectable projectable, EvalNode[] evalNodes)
throws PlanningException {
for (EvalNode e : evalNodes) {
@@ -649,7 +661,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
if (block.namedExprsMgr.isEvaluated(sortKeyRefNames[i])) {
column = block.namedExprsMgr.getTarget(sortKeyRefNames[i]).getNamedColumn();
} else {
- throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs));
+ throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs));
}
annotatedSortSpecs[i] = new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst());
}
@@ -866,7 +878,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
} else if (block.namedExprsMgr.isEvaluated(refName)) {
column = block.namedExprsMgr.getTarget(refName).getNamedColumn();
} else {
- throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs));
+ throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs));
}
annotatedSortSpecs.add(new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst()));
}
@@ -1175,13 +1187,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Schema joinSchema = new Schema();
Schema commons = SchemaUtil.getNaturalJoinColumns(left.getOutSchema(), right.getOutSchema());
joinSchema.addColumns(commons);
- for (Column c : left.getOutSchema().getColumns()) {
+ for (Column c : left.getOutSchema().getRootColumns()) {
if (!joinSchema.contains(c.getQualifiedName())) {
joinSchema.addColumn(c);
}
}
- for (Column c : right.getOutSchema().getColumns()) {
+ for (Column c : right.getOutSchema().getRootColumns()) {
if (!joinSchema.contains(c.getQualifiedName())) {
joinSchema.addColumn(c);
}
@@ -1199,7 +1211,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Column leftJoinKey;
Column rightJoinKey;
- for (Column common : commons.getColumns()) {
+ for (Column common : commons.getRootColumns()) {
leftJoinKey = leftSchema.getColumn(common.getQualifiedName());
rightJoinKey = rightSchema.getColumn(common.getQualifiedName());
equiQual = new BinaryEval(EvalType.EQUAL,
@@ -1312,7 +1324,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
private static LinkedHashSet<Target> createFieldTargetsFromRelation(QueryBlock block, RelationNode relationNode,
Set<String> newlyEvaluatedRefNames) {
LinkedHashSet<Target> targets = Sets.newLinkedHashSet();
- for (Column column : relationNode.getLogicalSchema().getColumns()) {
+ for (Column column : relationNode.getLogicalSchema().getAllColumns()) {
+
+ // TODO - Currently, EvalNode has DataType as a return type. So, RECORD cannot be used for any target.
+ // The following line is a kind of hack, preventing RECORD to be used for target in the logical planning phase.
+ // This problem should be resolved after TAJO-1402.
+ if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) {
+ continue;
+ }
+
String aliasName = block.namedExprsMgr.checkAndGetIfAliasedColumn(column.getQualifiedName());
if (aliasName != null) {
targets.add(new Target(new FieldEval(column), aliasName));
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
index 153a150..eddb022 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
@@ -26,6 +26,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import static org.apache.tajo.common.TajoDataTypes.DataType;
@@ -88,7 +89,7 @@ public class RowConstantEval extends EvalNode {
}
public String toString() {
- return TUtil.arrayToString(values);
+ return StringUtils.join(values);
}
public void preOrder(EvalNodeVisitor visitor) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
index 0b60d14..a39d303 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
@@ -26,6 +26,7 @@ import org.apache.tajo.datum.Datum;
import org.apache.tajo.plan.function.FunctionContext;
import org.apache.tajo.plan.logical.WindowSpec;
import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.Arrays;
@@ -120,7 +121,7 @@ public class WindowFunctionEval extends AggregationFunctionCallEval implements C
sb.append(funcDesc.getFunctionName()).append("(").append(isDistinct() ? " distinct" : "").append(sb)
.append(")");
if (hasSortSpecs()) {
- sb.append("ORDER BY ").append(TUtil.arrayToString(sortSpecs));
+ sb.append("ORDER BY ").append(StringUtils.join(sortSpecs));
}
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
index 878553f..079fc3e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
@@ -478,7 +478,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
@@ -512,7 +512,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -532,7 +532,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -552,7 +552,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext(functionContext);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -573,7 +573,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
return outputHandler.getPartialResultString();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -595,7 +595,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
index e4d6122..862cb8a 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
@@ -26,7 +26,7 @@ import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.expr.AlgebraicUtil;
import org.apache.tajo.plan.logical.*;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.*;
@@ -59,7 +59,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
for (LogicalNode relation : remainRelations) {
Collection <String> relationStrings = PlannerUtil.getRelationLineageWithinQueryBlock(plan, relation);
List<JoinEdge> joinEdges = new ArrayList<JoinEdge>();
- String relationCollection = TUtil.collectionToString(relationStrings, ",");
+ String relationCollection = StringUtils.join(relationStrings, ",");
List<JoinEdge> joinEdgesForGiven = joinGraph.getIncomingEdges(relationCollection);
if (joinEdgesForGiven != null) {
joinEdges.addAll(joinEdgesForGiven);
@@ -236,7 +236,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
// If outer is outer join, make edge key using all relation names in outer.
SortedSet<String> relationNames =
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer));
- String outerEdgeKey = TUtil.collectionToString(relationNames, ", ");
+ String outerEdgeKey = StringUtils.join(relationNames, ", ");
for (String innerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner)) {
if (graph.hasEdge(outerEdgeKey, innerName)) {
JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, innerName);
@@ -257,7 +257,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
relationNames =
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner));
- outerEdgeKey = TUtil.collectionToString(relationNames, ", ");
+ outerEdgeKey = StringUtils.join(relationNames, ", ");
for (String outerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer)) {
if (graph.hasEdge(outerEdgeKey, outerName)) {
JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, outerName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
index ce06748..fb4fae1 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
@@ -22,7 +22,7 @@ import com.google.common.collect.Sets;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.Collections;
import java.util.Set;
@@ -70,6 +70,6 @@ public class JoinEdge {
}
public String toString() {
- return leftRelation + " " + joinType + " " + rightRelation + " ON " + TUtil.collectionToString(joinQual, ", ");
+ return leftRelation + " " + joinType + " " + rightRelation + " ON " + StringUtils.join(joinQual, ", ");
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
index 72e9b1d..9ae5245 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Sets;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.graph.SimpleUndirectedGraph;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.NamedExprsManager;
@@ -33,7 +34,6 @@ import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.expr.EvalTreeUtil;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.plan.logical.RelationNode;
-import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -111,8 +111,8 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> {
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, joinNode.getRightChild()));
addEdge(
- TUtil.collectionToString(leftNodeRelationName, ", "),
- TUtil.collectionToString(rightNodeRelationName, ", "),
+ StringUtils.join(leftNodeRelationName, ", "),
+ StringUtils.join(rightNodeRelationName, ", "),
edge);
Set<EvalNode> allInOneCnf = new HashSet<EvalNode>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
index 61ce789..fb19f10 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
@@ -19,12 +19,12 @@
package org.apache.tajo.plan.logical;
import com.google.gson.annotations.Expose;
-
import org.apache.tajo.catalog.Column;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.AggregationFunctionCallEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.ArrayList;
@@ -150,7 +150,7 @@ public class DistinctGroupbyNode extends UnaryNode implements Projectable, Clone
public String toString() {
StringBuilder sb = new StringBuilder("Distinct GroupBy (");
if (groupingColumns != null && groupingColumns.length > 0) {
- sb.append("grouping set=").append(TUtil.arrayToString(groupingColumns));
+ sb.append("grouping set=").append(StringUtils.join(groupingColumns));
sb.append(", ");
}
for (GroupbyNode eachNode: subGroupbyPlan) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
index 3aaf5d0..0632f68 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
@@ -28,6 +28,7 @@ import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class EvalExprNode extends LogicalNode implements Projectable {
@@ -69,7 +70,7 @@ public class EvalExprNode extends LogicalNode implements Projectable {
@Override
public String toString() {
- return "EvalExprNode (" + TUtil.arrayToString(exprs) + ")";
+ return "EvalExprNode (" + StringUtils.join(exprs) + ")";
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
index 730f705..23a9154 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
@@ -18,18 +18,18 @@
package org.apache.tajo.plan.logical;
-import java.util.Arrays;
-
import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
-
import org.apache.tajo.catalog.Column;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.AggregationFunctionCallEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
+import java.util.Arrays;
+
public class GroupbyNode extends UnaryNode implements Projectable, Cloneable {
/** Grouping key sets */
@Expose private Column [] groupingKeys = PlannerUtil.EMPTY_COLUMNS;
@@ -110,11 +110,11 @@ public class GroupbyNode extends UnaryNode implements Projectable, Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("GroupBy (");
if (groupingKeys != null && groupingKeys.length > 0) {
- sb.append("grouping set=").append(TUtil.arrayToString(groupingKeys));
+ sb.append("grouping set=").append(StringUtils.join(groupingKeys));
sb.append(", ");
}
if (hasAggFunctions()) {
- sb.append("funcs=").append(TUtil.arrayToString(aggrFunctions));
+ sb.append("funcs=").append(StringUtils.join(aggrFunctions));
}
sb.append(")");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
index 3c9d497..ef6734e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
@@ -25,6 +25,7 @@ import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class ProjectionNode extends UnaryNode implements Projectable {
@@ -70,7 +71,7 @@ public class ProjectionNode extends UnaryNode implements Projectable {
public String toString() {
StringBuilder sb = new StringBuilder("Projection (distinct=").append(distinct);
if (targets != null) {
- sb.append(", exprs=").append(TUtil.arrayToString(targets)).append(")");
+ sb.append(", exprs=").append(StringUtils.join(targets)).append(")");
}
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
index 7e335b0..ced9a36 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
@@ -43,7 +43,19 @@ public abstract class RelationNode extends LogicalNode {
public abstract String getTableName();
+ /**
+ * Return a full qualified table name (i.e., dbname.table_name)
+ *
+ * @return A full qualified table name
+ */
public abstract String getCanonicalName();
+ /**
+ * Return a logical schema, meaning physically stored columns and virtual columns.
+ * Since partition keys in the column partition are not physically stored to files or tables,
+ * we call the partition keys virtual columns.
+ *
+ * @return A logical schema
+ */
public abstract Schema getLogicalSchema();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
index 8215f51..7487009 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
@@ -107,7 +108,7 @@ public class ShuffleFileWriteNode extends PersistentStoreNode implements Cloneab
}
sb.append(", part number=").append(numOutputs);
if (shuffleKeys != null) {
- sb.append(", keys: ").append(TUtil.arrayToString(shuffleKeys));
+ sb.append(", keys: ").append(StringUtils.join(shuffleKeys));
}
sb.append(")");
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
index 0166ef8..ef61d51 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
@@ -20,7 +20,7 @@ package org.apache.tajo.plan.logical;
import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.List;
@@ -52,7 +52,7 @@ public class TruncateTableNode extends LogicalNode {
@Override
public String toString() {
- return "TruncateTable (table=" + TUtil.collectionToString(tableNames, ", ") + ")";
+ return "TruncateTable (table=" + StringUtils.join(tableNames, ", ") + ")";
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
index 3f624f6..1bee34c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
@@ -26,6 +26,7 @@ import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.WindowFunctionEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class WindowAggNode extends UnaryNode implements Projectable, Cloneable {
@@ -114,14 +115,14 @@ public class WindowAggNode extends UnaryNode implements Projectable, Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("WinAgg (");
if (hasPartitionKeys()) {
- sb.append("partition keys=").append(TUtil.arrayToString(partitionKeys));
+ sb.append("partition keys=").append(StringUtils.join(partitionKeys));
sb.append(", ");
}
if (hasAggFunctions()) {
- sb.append("funcs=").append(TUtil.arrayToString(windowFuncs));
+ sb.append("funcs=").append(StringUtils.join(windowFuncs));
}
if (hasSortSpecs()) {
- sb.append("sort=").append(TUtil.arrayToString(sortSpecs));
+ sb.append("sort=").append(StringUtils.join(sortSpecs));
}
sb.append(")");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
index 2c55c81..633b30e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
@@ -18,11 +18,14 @@
package org.apache.tajo.plan.nameresolver;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.algebra.Relation;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.NestedPathUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.exception.NoSuchColumnException;
import org.apache.tajo.plan.algebra.AmbiguousFieldException;
@@ -31,14 +34,30 @@ import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.plan.logical.RelationNode;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
-import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
- * NameResolver utility
+ * Column name resolution utility. A SQL statement can include many kinds of column names,
+ * defined in different ways. Some column name indicates just a column in a relation.
+ * Another column name includes alias table name or alias column name, derived from some expression.
+ *
+ * This utility ensures that each column name is derived from valid and accessible column, and
+ * it also finds the exact data type of the column.
+ *
+ * Terminology:
+ * <ul>
+ * <li>Qualifier: database name, table name, or both included in a column name</li>
+ * <li>Simple name: just column name without any qualifier</li>
+ * <li>Alias name: another name to shortly specify a certain column</li>
+ * <li>Fully qualified name: a column name with database name and table name</li>
+ * <li>Canonical name: a fully qualified name, but its simple name is aliased name.</li>
+ * </ul>
*/
public abstract class NameResolver {
@@ -51,30 +70,44 @@ public abstract class NameResolver {
resolverMap.put(NameResolvingMode.LEGACY, new ResolverByLegacy());
}
+ public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
+ NameResolvingMode mode) throws PlanningException {
+ if (!resolverMap.containsKey(mode)) {
+ throw new PlanningException("Unsupported name resolving level: " + mode.name());
+ }
+ return resolverMap.get(mode).resolve(plan, block, column);
+ }
+
abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
throws PlanningException;
/**
- * Try to find the database name
+ * Guess a relation from a table name regardless of whether the given name is qualified or not.
*
* @param block the current block
- * @param tableName The table name
- * @return The found database name
+ * @param tableName The table name which can be either qualified or not.
+ * @return A corresponding relation
* @throws PlanningException
*/
- public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException {
- List<String> found = new ArrayList<String>();
+ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tableName) throws PlanningException {
+ List<RelationNode> found = TUtil.newList();
+
for (RelationNode relation : block.getRelations()) {
- // check alias name or table name
- if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
+
+ // if a table name is qualified
+ if (relation.getCanonicalName().equals(tableName) || relation.getTableName().equals(tableName)) {
+ found.add(relation);
+
+ // if a table name is not qualified
+ } else if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) {
- // obtain the database name
- found.add(CatalogUtil.extractQualifier(relation.getTableName()));
+ found.add(relation);
}
}
if (found.size() == 0) {
return null;
+
} else if (found.size() > 1) {
throw new PlanningException("Ambiguous table name \"" + tableName + "\"");
}
@@ -82,12 +115,26 @@ public abstract class NameResolver {
return found.get(0);
}
- public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
- NameResolvingMode mode) throws PlanningException {
- if (!resolverMap.containsKey(mode)) {
- throw new PlanningException("Unsupported name resolving level: " + mode.name());
+ /**
+ * Find relations such that its schema contains a given column
+ *
+ * @param block the current block
+ * @param columnName The column name to find relation
+ * @return relations including a given column
+ * @throws PlanningException
+ */
+ public static Collection<RelationNode> lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName)
+ throws PlanningException {
+
+ Set<RelationNode> found = TUtil.newHashSet();
+
+ for (RelationNode rel : block.getRelations()) {
+ if (rel.getLogicalSchema().contains(columnName)) {
+ found.add(rel);
+ }
}
- return resolverMap.get(mode).resolve(plan, block, column);
+
+ return found;
}
/**
@@ -107,7 +154,7 @@ public abstract class NameResolver {
String canonicalName;
if (columnRef.hasQualifier()) {
- Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef);
qualifier = normalized.getFirst();
canonicalName = normalized.getSecond();
@@ -121,8 +168,8 @@ public abstract class NameResolver {
// Please consider a query case:
// select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey;
//
- // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still can be used.
- // The below code makes it available. Otherwise, it cannot find any match in the relation schema.
+ // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still should be available.
+ // The below code makes it possible. Otherwise, it cannot find any match in the relation schema.
if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) {
canonicalName =
CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName));
@@ -133,7 +180,7 @@ public abstract class NameResolver {
return column;
} else {
- return resolveFromAllRelsInBlock(block, columnRef);
+ return lookupColumnFromAllRelsInBlock(block, columnRef.getName());
}
}
@@ -162,18 +209,22 @@ public abstract class NameResolver {
}
/**
- * It tries to find a full qualified column name from all relations in the current block.
+ * Lookup a column among all relations in the current block from a column name.
+ *
+ * It assumes that <code>columnName</code> is not any qualified name.
*
* @param block The current query block
- * @param columnRef The column reference to be found
+ * @param columnName The column reference to be found
* @return The found column
*/
- static Column resolveFromAllRelsInBlock(LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef) throws VerifyException {
+ static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block,
+ String columnName) throws VerifyException {
+ Preconditions.checkArgument(CatalogUtil.isSimpleIdentifier(columnName));
+
List<Column> candidates = TUtil.newList();
for (RelationNode rel : block.getRelations()) {
- Column found = rel.getLogicalSchema().getColumn(columnRef.getName());
+ Column found = rel.getLogicalSchema().getColumn(columnName);
if (found != null) {
candidates.add(found);
}
@@ -240,39 +291,100 @@ public abstract class NameResolver {
}
/**
- * It returns a pair of names, which the first value is ${database}.${table} and the second value
- * is a simple column name.
+ * Lookup a qualifier and a canonical name of column.
+ *
+ * It returns a pair of names, which the first value is the qualifier ${database}.${table} and
+ * the second value is column's simple name.
*
* @param block The current block
* @param columnRef The column name
* @return A pair of normalized qualifier and column name
* @throws PlanningException
*/
- static Pair<String, String> normalizeQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef)
+ static Pair<String, String> lookupQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef)
throws PlanningException {
- String qualifier;
- String canonicalName;
+ Preconditions.checkArgument(columnRef.hasQualifier(), "ColumnReferenceExpr must be qualified.");
+
+ String [] qualifierParts = columnRef.getQualifier().split("\\.");
- if (CatalogUtil.isFQTableName(columnRef.getQualifier())) {
- qualifier = columnRef.getQualifier();
- canonicalName = columnRef.getCanonicalName();
+ // This method assumes that column name consists of two or more dot chained names.
+ // In this case, there must be three cases as follows:
+ //
+ // - dbname.tbname.column_name.nested_field...
+ // - tbname.column_name.nested_field...
+ // - column.nested_fieldX...
+
+ Set<RelationNode> guessedRelations = TUtil.newHashSet();
+
+ // this position indicates the index of column name in qualifierParts;
+ // It must be 0 or more because a qualified column is always passed to lookupQualifierAndCanonicalName().
+ int columnNamePosition = -1;
+
+ // check for dbname.tbname.column_name.nested_field
+ if (qualifierParts.length >= 2) {
+ RelationNode rel = lookupTable(block, CatalogUtil.buildFQName(qualifierParts[0], qualifierParts[1]));
+ if (rel != null) {
+ guessedRelations.add(rel);
+ columnNamePosition = 2;
+ }
+ }
+
+ // check for tbname.column_name.nested_field
+ if (qualifierParts.length >= 1) {
+ RelationNode rel = lookupTable(block, qualifierParts[0]);
+ if (rel != null) {
+ guessedRelations.add(rel);
+ columnNamePosition = 1;
+ }
+ }
+
+ // column.nested_fieldX...
+ if (guessedRelations.size() == 0 && qualifierParts.length == 1) {
+ Collection<RelationNode> rels = lookupTableByColumns(block, qualifierParts[0]);
+
+ if (rels.size() > 1) {
+ throw new AmbiguousFieldException(columnRef.getCanonicalName());
+ }
+
+ if (rels.size() == 1) {
+ guessedRelations.addAll(rels);
+ columnNamePosition = 0;
+ }
+ }
+
+ // throw exception if no column cannot be founded or two or more than columns are founded
+ if (guessedRelations.size() == 0) {
+ throw new NoSuchColumnException(columnRef.getQualifier());
+ } else if (guessedRelations.size() > 1) {
+ throw new AmbiguousFieldException(columnRef.getCanonicalName());
+ }
+
+ String qualifier = guessedRelations.iterator().next().getCanonicalName();
+ String columnName = "";
+
+ if (columnNamePosition >= qualifierParts.length) { // if there is no column in qualifierParts
+ columnName = columnRef.getName();
} else {
- String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier());
- if (resolvedDatabaseName == null) {
- throw new NoSuchColumnException(columnRef.getQualifier());
+ // join a column name and its nested field names
+ columnName = qualifierParts[columnNamePosition];
+
+ // if qualifierParts include nested field names
+ if (qualifierParts.length > columnNamePosition) {
+ columnName += StringUtils.join(qualifierParts, "/", columnNamePosition + 1, qualifierParts.length);
}
- qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier());
- canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
+
+ // columnRef always has a leaf field name.
+ columnName += "/" + columnRef.getName();
}
- return new Pair<String, String>(qualifier, canonicalName);
+ return new Pair<String, String>(qualifier, columnName);
}
static Column ensureUniqueColumn(List<Column> candidates) throws VerifyException {
if (candidates.size() == 1) {
return candidates.get(0);
- } else if (candidates.size() > 2) {
+ } else if (candidates.size() > 1) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (Column column : candidates) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
index 19f39dd..0a665ab 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
@@ -48,12 +48,10 @@ public class ResolverByLegacy extends NameResolver {
private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
ColumnReferenceExpr columnRef) throws PlanningException {
final String qualifier;
- String canonicalName;
final String qualifiedName;
- Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef);
qualifier = normalized.getFirst();
- canonicalName = normalized.getSecond();
qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
Column found = resolveFromRelsWithinBlock(plan, block, columnRef);
@@ -84,7 +82,7 @@ public class ResolverByLegacy extends NameResolver {
&& currentNode.getType() != NodeType.TABLE_SUBQUERY) {
List<Column> candidates = TUtil.newList();
if (block.getNamedExprsManager().isAliased(qualifiedName)) {
- String alias = block.getNamedExprsManager().getAlias(canonicalName);
+ String alias = block.getNamedExprsManager().getAlias(qualifiedName);
found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY);
if (found != null) {
candidates.add(found);
@@ -101,7 +99,7 @@ public class ResolverByLegacy extends NameResolver {
static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
ColumnReferenceExpr columnRef)throws PlanningException {
- Column found = resolveFromAllRelsInBlock(block, columnRef);
+ Column found = lookupColumnFromAllRelsInBlock(block, columnRef.getName());
if (found != null) {
return found;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index dc6b8ef..cc54a22 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@ -726,7 +726,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
// find aggregation column
Set<Column> groupingColumns = TUtil.newHashSet(groupByNode.getGroupingColumns());
Set<String> aggrFunctionOutColumns = TUtil.newHashSet();
- for (Column column : groupByNode.getOutSchema().getColumns()) {
+ for (Column column : groupByNode.getOutSchema().getRootColumns()) {
if (!groupingColumns.contains(column)) {
aggrFunctionOutColumns.add(column.getQualifiedName());
}
@@ -842,7 +842,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
TableDesc table = scanNode.getTableDesc();
boolean hasQualifiedName = false;
if (table.hasPartition()) {
- for (Column c: table.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column c: table.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionColumns.add(c.getQualifiedName());
hasQualifiedName = c.hasQualifier();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index d8b6380..4b9fd48 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -224,7 +224,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod();
Schema paritionValuesSchema = new Schema();
- for (Column column : partitionDesc.getExpressionSchema().getColumns()) {
+ for (Column column : partitionDesc.getExpressionSchema().getRootColumns()) {
paritionValuesSchema.addColumn(column);
}
@@ -237,7 +237,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
// add qualifier to schema for qual
paritionValuesSchema.setQualifier(scanNode.getCanonicalName());
- for (Column column : paritionValuesSchema.getColumns()) {
+ for (Column column : paritionValuesSchema.getRootColumns()) {
for (EvalNode simpleExpr : conjunctiveForms) {
if (checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) {
indexablePredicateSet.add(simpleExpr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 763f938..3b83ded 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -146,7 +146,7 @@ public class PlannerUtil {
if (EvalTreeUtil.checkIfPartitionSelection(node, partSchema)) {
prefixPartitionWhere = true;
boolean isPrefix = true;
- for (Column c : partSchema.getColumns()) {
+ for (Column c : partSchema.getRootColumns()) {
String value = EvalTreeUtil.getPartitionValue(node, c.getSimpleName());
if (isPrefix && value == null)
isPrefix = false;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
index 22b3351..c5ef8ef 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
@@ -23,6 +23,8 @@ import org.apache.tajo.OverridableConf;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.util.PlannerUtil;
@@ -55,7 +57,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
/**
* It checks if an output schema of a projectable node and target's output data types are equivalent to each other.
*/
- private static void verifyProjectableOutputSchema(Projectable node) throws PlanningException {
+ private static void verifyProjectableOutputSchema(Context context, Projectable node) throws PlanningException {
Schema outputSchema = node.getOutSchema();
Schema targetSchema = PlannerUtil.targetToSchema(node.getTargets());
@@ -66,9 +68,15 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
}
for (int i = 0; i < outputSchema.size(); i++) {
- if (!outputSchema.getColumn(i).getDataType().equals(targetSchema.getColumn(i).getDataType())) {
+ Column outputColumn = outputSchema.getColumn(i);
+
+ if (outputColumn.getDataType().getType() == Type.RECORD) {
+ context.state.addVerification("Projecting RECORD fields is not supported yet.");
+ }
+
+ if (!outputColumn.getDataType().equals(targetSchema.getColumn(i).getDataType())) {
Column targetColumn = targetSchema.getColumn(i);
- Column insertColumn = outputSchema.getColumn(i);
+ Column insertColumn = outputColumn;
throw new PlanningException("ERROR: " +
insertColumn.getSimpleName() + " is of type " + insertColumn.getDataType().getType().name() +
", but target column '" + targetColumn.getSimpleName() + "' is of type " +
@@ -86,7 +94,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(state.state, node, target.getEvalTree());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(state, node);
return node;
}
@@ -108,7 +116,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
GroupbyNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitGroupBy(context, plan, block, node, stack);
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -130,7 +138,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(context.state, node, node.getJoinQual());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -192,7 +200,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
}
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -209,7 +217,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(context.state, node, node.getQual());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
index 275ab3a..8717bda 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
@@ -83,7 +83,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
super.visitProjection(context, stack, expr);
Set<String> names = TUtil.newHashSet();
-
+
for (NamedExpr namedExpr : expr.getNamedExprs()) {
if (namedExpr.hasAlias()) {
@@ -302,8 +302,8 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
return null;
}
if (table.hasPartition()) {
- int columnSize = table.getSchema().getColumns().size();
- columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size();
+ int columnSize = table.getSchema().getRootColumns().size();
+ columnSize += table.getPartitionMethod().getExpressionSchema().getRootColumns().size();
if (projectColumnNum < columnSize) {
context.state.addVerification("INSERT has smaller expressions than target columns");
} else if (projectColumnNum > columnSize) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
index 66b3667..5423fd7 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
@@ -84,7 +84,7 @@ public class MergeScanner implements Scanner {
tableStats.setNumBytes(numBytes);
tableStats.setNumBlocks(fragments.size());
- for(Column eachColumn: schema.getColumns()) {
+ for(Column eachColumn: schema.getRootColumns()) {
ColumnStats columnStats = new ColumnStats(eachColumn);
tableStats.addColumnStat(columnStats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 33db798..9d69423 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -35,7 +35,7 @@ public class RowStoreUtil {
public static int[] getTargetIds(Schema inSchema, Schema outSchema) {
int[] targetIds = new int[outSchema.size()];
int i = 0;
- for (Column target : outSchema.getColumns()) {
+ for (Column target : outSchema.getRootColumns()) {
targetIds[i] = inSchema.getColumnId(target.getQualifiedName());
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
index 0356b19..7af8247 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
@@ -61,6 +61,15 @@ public interface Scanner extends SchemaObject, Closeable {
/**
* It returns if the projection is executed in the underlying scanner layer.
*
+ * If TRUE, the upper layers (i.e., SeqScanExec) assume that next()
+ * will return a tuple which contains only projected fields. In other words,
+ * the field number of a retrieved tuple is equivalent tothe number of targets.
+ *
+ * If FALSE, the upper layers assume that next() will return a tuple which
+ * contains projected fields and non-projected fields, padded by NullDatum.
+ * In other words, the width of tuple is equivalent to the field number
+ * of the table schema.
+ *
* @return true if this scanner can project the given columns.
*/
boolean isProjectable();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
index ce963c8..b8438e9 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -409,9 +409,7 @@ public abstract class StorageManager {
Class<? extends Scanner> scannerClass = getScannerClass(meta.getStoreType());
scanner = newScannerInstance(scannerClass, conf, schema, meta, fragment);
- if (scanner.isProjectable()) {
- scanner.setTarget(target.toArray());
- }
+ scanner.setTarget(target.toArray());
return scanner;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
index 4836352..0e3441b 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
@@ -60,7 +60,6 @@ public abstract class AbstractHBaseAppender implements Appender {
protected boolean[] isRowKeyMappings;
protected boolean[] isColumnKeys;
protected boolean[] isColumnValues;
- protected int[] rowKeyFieldIndexes;
protected int[] rowkeyColumnIndexes;
protected char rowKeyDelimiter;
@@ -107,7 +106,6 @@ public abstract class AbstractHBaseAppender implements Appender {
isColumnKeys = columnMapping.getIsColumnKeys();
isColumnValues = columnMapping.getIsColumnValues();
rowKeyDelimiter = columnMapping.getRowKeyDelimiter();
- rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes();
this.columnNum = schema.size();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
index c3094fd..e66a707 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
@@ -77,7 +77,7 @@ public class ColumnMapping {
String[] columnMappingTokens = columnMapping.split(",");
- if (columnMappingTokens.length != schema.getColumns().size()) {
+ if (columnMappingTokens.length != schema.getRootColumns().size()) {
throw new IOException("The number of mapped HBase columns is great than the number of Tajo table columns");
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
index c05c5bb..53ff9dc 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
@@ -39,10 +39,14 @@ public class HBaseBinarySerializerDeserializer {
datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes));
break;
case INT8:
- if (bytes.length == 4) {
- datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes));
+ if (bytes == null) {
+ datum = NullDatum.get();
} else {
- datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes));
+ if (bytes.length == 4) {
+ datum = DatumFactory.createInt8(Bytes.toInt(bytes));
+ } else {
+ datum = bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes));
+ }
}
break;
case FLOAT4:
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
index ab56252..9eb1d86 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage.hbase;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -87,7 +88,13 @@ public class HBaseScanner implements Scanner {
private char rowKeyDelimiter;
public HBaseScanner (Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException {
- this.conf = (TajoConf)conf;
+ Preconditions.checkNotNull(conf);
+ Preconditions.checkNotNull(schema);
+ Preconditions.checkNotNull(meta);
+ Preconditions.checkNotNull(fragment);
+ Preconditions.checkArgument(conf instanceof TajoConf);
+
+ this.conf = (TajoConf) conf;
this.schema = schema;
this.meta = meta;
this.fragment = (HBaseFragment)fragment;
@@ -102,11 +109,10 @@ public class HBaseScanner implements Scanner {
tableStats.setNumBytes(0);
tableStats.setNumBlocks(1);
}
- if (schema != null) {
- for(Column eachColumn: schema.getColumns()) {
- ColumnStats columnStats = new ColumnStats(eachColumn);
- tableStats.addColumnStat(columnStats);
- }
+
+ for (Column eachColumn : schema.getRootColumns()) {
+ ColumnStats columnStats = new ColumnStats(eachColumn);
+ tableStats.addColumnStat(columnStats);
}
scanFetchSize = Integer.parseInt(
@@ -203,9 +209,9 @@ public class HBaseScanner implements Scanner {
}
Result result = scanResults[scanResultIndex++];
- Tuple resultTuple = new VTuple(schema.size());
+ Tuple resultTuple = new VTuple(targetIndexes.length);
for (int i = 0; i < targetIndexes.length; i++) {
- resultTuple.put(targetIndexes[i], getDatum(result, targetIndexes[i]));
+ resultTuple.put(i, getDatum(result, targetIndexes[i]));
}
numRows++;
return resultTuple;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
index a9e5bde..f74245e 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
@@ -355,7 +355,7 @@ public class HBaseStorageManager extends StorageManager {
Collection<String> columnFamilies = columnMapping.getColumnFamilyNames();
//If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column.
if (columnFamilies.isEmpty()) {
- for (Column eachColumn: schema.getColumns()) {
+ for (Column eachColumn: schema.getRootColumns()) {
columnFamilies.add(eachColumn.getSimpleName());
}
}
@@ -735,7 +735,7 @@ public class HBaseStorageManager extends StorageManager {
for (String property : CONNECTION_PROPERTIES) {
String thisValue = this.properties.get(property);
String thatValue = that.properties.get(property);
- //noinspection StringEquality
+ // noinspection StringEquality
if (thisValue == thatValue) {
continue;
}
@@ -1121,7 +1121,7 @@ public class HBaseStorageManager extends StorageManager {
try {
HTableDescriptor hTableDesc = parseHTableDescriptor(tableMeta, cNode.getTableSchema());
- LOG.info("Delete table cause query failed:" + hTableDesc.getName());
+ LOG.info("Delete table cause query failed:" + new String(hTableDesc.getName()));
hAdmin.disableTable(hTableDesc.getName());
hAdmin.deleteTable(hTableDesc.getName());
} finally {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
index c1047d9..5fc96f1 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -528,7 +528,7 @@ public class CSVFile {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
index 038f0f4..0726125 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
@@ -69,7 +69,7 @@ public abstract class FileScanner implements Scanner {
}
if (schema != null) {
- for(Column eachColumn: schema.getColumns()) {
+ for(Column eachColumn: schema.getRootColumns()) {
ColumnStats columnStats = new ColumnStats(eachColumn);
tableStats.addColumnStat(columnStats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
index 8d425b4..439bee0 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
@@ -732,7 +732,7 @@ public class FileStorageManager extends StorageManager {
//In the case of partitioned table, we should return same partition key data files.
int partitionDepth = 0;
if (tableDesc.hasPartition()) {
- partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size();
+ partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size();
}
List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
index 51594df..26083a5 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
@@ -80,8 +80,7 @@ public class AvroScanner extends FileScanner {
avroSchema = AvroUtil.getAvroSchema(meta, conf);
avroFields = avroSchema.getFields();
- DatumReader<GenericRecord> datumReader =
- new GenericDatumReader<GenericRecord>(avroSchema);
+ DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(avroSchema);
SeekableInput input = new FsInput(fragment.getPath(), conf);
dataFileReader = new DataFileReader<GenericRecord>(input, datumReader);
super.init();
@@ -175,13 +174,13 @@ public class AvroScanner extends FileScanner {
return null;
}
- Tuple tuple = new VTuple(schema.size());
+ Tuple tuple = new VTuple(projectionMap.length);
GenericRecord record = dataFileReader.next();
for (int i = 0; i < projectionMap.length; ++i) {
int columnIndex = projectionMap[i];
Object value = record.get(columnIndex);
if (value == null) {
- tuple.put(columnIndex, NullDatum.get());
+ tuple.put(i, NullDatum.get());
continue;
}
@@ -196,28 +195,28 @@ public class AvroScanner extends FileScanner {
TajoDataTypes.Type tajoType = dataType.getType();
switch (avroType) {
case NULL:
- tuple.put(columnIndex, NullDatum.get());
+ tuple.put(i, NullDatum.get());
break;
case BOOLEAN:
- tuple.put(columnIndex, DatumFactory.createBool((Boolean)value));
+ tuple.put(i, DatumFactory.createBool((Boolean)value));
break;
case INT:
- tuple.put(columnIndex, convertInt(value, tajoType));
+ tuple.put(i, convertInt(value, tajoType));
break;
case LONG:
- tuple.put(columnIndex, DatumFactory.createInt8((Long)value));
+ tuple.put(i, DatumFactory.createInt8((Long)value));
break;
case FLOAT:
- tuple.put(columnIndex, DatumFactory.createFloat4((Float)value));
+ tuple.put(i, DatumFactory.createFloat4((Float)value));
break;
case DOUBLE:
- tuple.put(columnIndex, DatumFactory.createFloat8((Double)value));
+ tuple.put(i, DatumFactory.createFloat8((Double)value));
break;
case BYTES:
- tuple.put(columnIndex, convertBytes(value, tajoType, dataType));
+ tuple.put(i, convertBytes(value, tajoType, dataType));
break;
case STRING:
- tuple.put(columnIndex, convertString(value, tajoType));
+ tuple.put(i, convertString(value, tajoType));
break;
case RECORD:
throw new RuntimeException("Avro RECORD not supported.");
@@ -228,7 +227,7 @@ public class AvroScanner extends FileScanner {
case UNION:
throw new RuntimeException("Avro UNION not supported.");
case FIXED:
- tuple.put(columnIndex, new BlobDatum(((GenericFixed)value).bytes()));
+ tuple.put(i, new BlobDatum(((GenericFixed)value).bytes()));
break;
default:
throw new RuntimeException("Unknown type.");