You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2013/10/22 09:29:13 UTC
git commit: TAJO-270: Boolean datum compatible to apache hive. (jinho)
Updated Branches:
refs/heads/master b55918bd1 -> 85cff2e71
TAJO-270: Boolean datum compatible to apache hive. (jinho)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/85cff2e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/85cff2e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/85cff2e7
Branch: refs/heads/master
Commit: 85cff2e7101e6e059285a80268ba2d8aaaca00ec
Parents: b55918b
Author: jinossy <ji...@gmail.com>
Authored: Tue Oct 22 16:27:49 2013 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Tue Oct 22 16:27:49 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/tajo/datum/BooleanDatum.java | 2 +-
.../org/apache/tajo/datum/DatumFactory.java | 7 +---
.../tajo/engine/planner/LogicalPlanner.java | 3 +-
.../apache/tajo/master/querymaster/Query.java | 4 +-
.../java/org/apache/tajo/storage/CSVFile.java | 44 +++++++++++++++-----
.../java/org/apache/tajo/storage/LazyTuple.java | 2 +-
7 files changed, 44 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a088588..9bb67fe 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -59,6 +59,8 @@ Release 0.2.0 - unreleased
IMPROVEMENTS
+ TAJO-270: Boolean datum compatible to apache hive. (jinho)
+
TAJO-261: Rearrange default port numbers and config names. (hyunsik)
TAJO-236: Implement LogicalPlanVerifier to check if a logical plan is
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
index aeab5aa..bf4d83d 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/BooleanDatum.java
@@ -48,7 +48,7 @@ public class BooleanDatum extends Datum {
public BooleanDatum(byte[] bytes) {
this(bytes[0]);
}
-
+
public boolean asBool() {
return val;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
index e783666..5268ca5 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DatumFactory.java
@@ -94,12 +94,7 @@ public class DatumFactory {
public static NullDatum createNullDatum() {
return NullDatum.get();
}
-
- public static BooleanDatum createBool(String val) {
- boolean boolVal = val.equalsIgnoreCase("t");
- return new BooleanDatum(boolVal);
- }
-
+
public static BooleanDatum createBool(byte val) {
boolean boolVal = val == 0x01;
return new BooleanDatum(boolVal);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 9f7fe25..84cbe14 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -915,7 +915,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
LiteralValue literal = (LiteralValue) expr;
switch (literal.getValueType()) {
case Boolean:
- return new ConstEval(DatumFactory.createBool(literal.getValue()));
+ char val = literal.getValue().charAt(0);
+ return new ConstEval(DatumFactory.createBool(val == 't' || val == 'T'));
case String:
return new ConstEval(DatumFactory.createText(literal.getValue()));
case Unsigned_Integer:
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 715f00c..ce945b4 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.*;
+import org.apache.tajo.catalog.statistics.TableStat;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
@@ -382,9 +383,10 @@ public class Query implements EventHandler<QueryEvent> {
try {
FileSystem fs = finalOutputDir.getFileSystem(query.systemConf);
ContentSummary directorySummary = fs.getContentSummary(finalOutputDir);
+ if(meta.getStat() == null) meta.setStat(new TableStat());
meta.getStat().setNumBytes(directorySummary.getLength());
} catch (IOException e) {
- LOG.error(e);
+ LOG.error(e.getMessage(), e);
}
TableDesc outputTableDesc = new TableDescImpl(outputTableName, meta, finalOutputDir);
TableDesc finalTableDesc = outputTableDesc;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 58d772e..550148c 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -47,11 +47,15 @@ import java.io.*;
import java.util.Arrays;
public class CSVFile {
+ public static byte[] trueBytes = "true".getBytes();
+ public static byte[] falseBytes = "false".getBytes();
+
public static final String DELIMITER = "csvfile.delimiter";
public static final String NULL = "csvfile.null"; //read only
public static final String DELIMITER_DEFAULT = "|";
public static final byte LF = '\n';
public static int EOF = -1;
+
private static final Log LOG = LogFactory.getLog(CSVFile.class);
public static class CSVAppender extends FileAppender {
@@ -135,26 +139,46 @@ public class CSVFile {
LazyTuple lTuple = (LazyTuple)tuple;
for (int i = 0; i < colNum; i++) {
TajoDataTypes.DataType dataType = schema.getColumn(i).getDataType();
- datum = tuple.get(i);
switch (dataType.getType()) {
- case TEXT:
- outputStream.write(datum.asTextBytes());
+ case TEXT: {
+ datum = tuple.get(i);
+ if (datum instanceof NullDatum) {
+ outputStream.write(nullChars);
+ } else {
+ outputStream.write(datum.asTextBytes());
+ }
+ break;
+ }
+ case CHAR: {
+ datum = tuple.get(i);
+ if (datum instanceof NullDatum) {
+ outputStream.write(nullChars);
+ } else {
+ byte[] pad = new byte[dataType.getLength() - datum.size()];
+ outputStream.write(datum.asTextBytes());
+ outputStream.write(pad);
+ }
break;
- case CHAR:
- byte[] pad = new byte[dataType.getLength() - datum.size()];
- outputStream.write(datum.asTextBytes());
- outputStream.write(pad);
+ }
+ case BOOLEAN: {
+ datum = tuple.get(i);
+ if (datum instanceof NullDatum) {
+ //null datum is zero length byte array
+ } else {
+ outputStream.write(datum.asBool() ? trueBytes : falseBytes); //Compatibility with Apache Hive
+ }
break;
+ }
case NULL:
- outputStream.write(nullChars);
break;
case PROTOBUF:
+ datum = tuple.get(i);
ProtobufDatum protobufDatum = (ProtobufDatum) datum;
protobufJsonFormat.print(protobufDatum.get(), outputStream);
break;
default:
- outputStream.write(lTuple.getTextBytes(i));
+ outputStream.write(lTuple.getTextBytes(i)); //better usage for insertion to table of lazy tuple
break;
}
@@ -179,7 +203,7 @@ public class CSVFile {
col = schema.getColumn(i);
switch (col.getDataType().getType()) {
case BOOLEAN:
- outputStream.write(tuple.getBoolean(i).asTextBytes());
+ outputStream.write(tuple.getBoolean(i).asBool() ? trueBytes : falseBytes); //Compatibility with Apache Hive
break;
case BIT:
outputStream.write(tuple.getByte(i).asTextBytes());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/85cff2e7/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
index 1d1d5fe..a395ea4 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/LazyTuple.java
@@ -306,7 +306,7 @@ public class LazyTuple implements Tuple {
private Datum createByTextBytes(TajoDataTypes.DataType type, byte [] val) {
switch (type.getType()) {
case BOOLEAN:
- return isNotNull(val) ? DatumFactory.createBool(new String(val)) : NullDatum.get();
+ return isNotNull(val) ? DatumFactory.createBool(val[0] == 't' || val[0] == 'T') : NullDatum.get();
case INT2:
return isNotNull(val) ? DatumFactory.createInt2(new String(val)) : NullDatum.get();
case INT4: