You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hj...@apache.org on 2014/12/05 09:21:29 UTC
[25/29] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into hbase_storage
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into hbase_storage
Conflicts:
CHANGES
tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
tajo-storage/pom.xml
tajo-storage/src/test/java/org/apache/tajo/storage/TestLineReader.java
tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileSystems.java
tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/s3/INode.java
tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/s3/InMemoryFileSystemStore.java
tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/s3/S3OutputStream.java
tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/s3/SmallBlockS3FileSystem.java
tajo-storage/tajo-storage-hdfs/src/test/resources/testVariousTypes.avsc
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/940546a0
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/940546a0
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/940546a0
Branch: refs/heads/hbase_storage
Commit: 940546a06143d9ba508b4d82a921b98b52116d0e
Parents: dfd7f99 1cdbe46
Author: HyoungJun Kim <ba...@babokim-MacBook-Pro.local>
Authored: Fri Dec 5 14:38:36 2014 +0900
Committer: HyoungJun Kim <ba...@babokim-MacBook-Pro.local>
Committed: Fri Dec 5 14:38:36 2014 +0900
----------------------------------------------------------------------
CHANGES | 28 ++
.../org/apache/tajo/algebra/CreateTable.java | 2 +-
.../main/java/org/apache/tajo/algebra/Expr.java | 2 +-
.../org/apache/tajo/catalog/AlterTableDesc.java | 2 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 9 +-
.../java/org/apache/tajo/catalog/Column.java | 2 +-
.../org/apache/tajo/catalog/FunctionDesc.java | 2 +-
.../java/org/apache/tajo/catalog/Schema.java | 3 +-
.../org/apache/tajo/catalog/SchemaUtil.java | 8 +
.../java/org/apache/tajo/catalog/SortSpec.java | 2 +-
.../java/org/apache/tajo/catalog/TableDesc.java | 2 +-
.../java/org/apache/tajo/catalog/TableMeta.java | 2 +-
.../tajo/catalog/json/CatalogGsonHelper.java | 2 +-
.../tajo/catalog/json/FunctionAdapter.java | 4 +-
.../tajo/catalog/json/TableMetaAdapter.java | 4 +-
.../tajo/catalog/partition/PartitionDesc.java | 2 +-
.../catalog/partition/PartitionMethodDesc.java | 2 +-
.../tajo/catalog/statistics/ColumnStats.java | 2 +-
.../tajo/catalog/statistics/TableStats.java | 2 +-
.../java/org/apache/tajo/function/Function.java | 2 +-
.../src/main/proto/CatalogProtos.proto | 3 +-
.../org/apache/tajo/client/ResultSetUtil.java | 8 +
.../java/org/apache/tajo/conf/TajoConf.java | 12 +-
.../main/java/org/apache/tajo/datum/Datum.java | 4 +-
.../apache/tajo/json/ClassNameSerializer.java | 47 ---
.../org/apache/tajo/json/CommonGsonHelper.java | 92 ------
.../org/apache/tajo/json/DataTypeAdapter.java | 63 ----
.../java/org/apache/tajo/json/DatumAdapter.java | 76 -----
.../java/org/apache/tajo/json/GsonHelper.java | 49 ---
.../java/org/apache/tajo/json/GsonObject.java | 23 --
.../org/apache/tajo/json/GsonSerDerAdapter.java | 25 --
.../org/apache/tajo/json/PathSerializer.java | 41 ---
.../apache/tajo/storage/StorageConstants.java | 15 +-
.../java/org/apache/tajo/storage/VTuple.java | 4 +-
.../tajo/storage/json/ClassNameSerializer.java | 47 +++
.../tajo/storage/json/CommonGsonHelper.java | 92 ++++++
.../tajo/storage/json/DataTypeAdapter.java | 63 ++++
.../apache/tajo/storage/json/DatumAdapter.java | 76 +++++
.../apache/tajo/storage/json/GsonHelper.java | 49 +++
.../apache/tajo/storage/json/GsonObject.java | 23 ++
.../tajo/storage/json/GsonSerDerAdapter.java | 25 ++
.../tajo/storage/json/PathSerializer.java | 41 +++
.../java/org/apache/tajo/util/KeyValueSet.java | 4 +-
.../org/apache/tajo/util/ReflectionUtil.java | 4 +-
.../org/apache/tajo/datum/TestBytesDatum.java | 2 +-
.../org/apache/tajo/datum/TestDateDatum.java | 2 +-
.../org/apache/tajo/datum/TestInet4Datum.java | 2 +-
.../org/apache/tajo/datum/TestTimeDatum.java | 2 +-
.../apache/tajo/datum/TestTimestampDatum.java | 5 +-
tajo-core/pom.xml | 1 +
.../apache/tajo/engine/json/CoreGsonHelper.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 63 +++-
.../engine/planner/physical/SeqScanExec.java | 7 +-
.../org/apache/tajo/master/ContainerProxy.java | 10 +-
.../tajo/master/DefaultTaskScheduler.java | 18 +-
.../org/apache/tajo/master/GlobalEngine.java | 7 +-
.../tajo/master/LaunchTaskRunnersEvent.java | 5 +-
.../apache/tajo/master/LazyTaskScheduler.java | 10 +-
.../apache/tajo/master/TajoContainerProxy.java | 22 +-
.../java/org/apache/tajo/master/TajoMaster.java | 2 +-
.../apache/tajo/master/TajoMasterService.java | 5 +-
.../tajo/master/TaskRunnerGroupEvent.java | 8 +-
.../apache/tajo/master/cluster/ServerName.java | 123 --------
.../tajo/master/container/TajoContainer.java | 173 ++++++++++
.../tajo/master/container/TajoContainerId.java | 172 ++++++++++
.../master/container/TajoConverterUtils.java | 263 ++++++++++++++++
.../master/container/TajoRecordFactory.java | 31 ++
.../container/TajoRecordFactoryPBImpl.java | 104 ++++++
.../container/TajoRecordFactoryProvider.java | 70 +++++
.../tajo/master/container/TajoRecords.java | 39 +++
.../impl/pb/TajoContainerIdPBImpl.java | 100 ++++++
.../tajo/master/event/LocalTaskEvent.java | 9 +-
.../event/QueryUnitAttemptScheduleEvent.java | 10 +-
.../event/SubQueryContainerAllocationEvent.java | 8 +-
.../master/event/TaskAttemptAssignedEvent.java | 8 +-
.../tajo/master/event/TaskRequestEvent.java | 8 +-
.../apache/tajo/master/querymaster/Query.java | 18 +-
.../master/querymaster/QueryInProgress.java | 3 +-
.../tajo/master/querymaster/QueryInfo.java | 2 +-
.../querymaster/QueryMasterManagerService.java | 4 +-
.../master/querymaster/QueryMasterTask.java | 29 +-
.../master/querymaster/QueryUnitAttempt.java | 6 +-
.../tajo/master/querymaster/SubQuery.java | 18 +-
.../apache/tajo/master/rm/TajoRMContext.java | 8 +-
.../tajo/master/rm/TajoWorkerContainer.java | 15 +-
.../tajo/master/rm/TajoWorkerContainerId.java | 53 ++--
.../master/rm/TajoWorkerResourceManager.java | 63 ++--
.../tajo/master/rm/WorkerResourceManager.java | 4 +-
.../apache/tajo/util/history/QueryHistory.java | 2 +-
.../tajo/util/history/QueryUnitHistory.java | 2 +-
.../tajo/util/history/SubQueryHistory.java | 2 +-
.../tajo/webapp/QueryExecutorServlet.java | 16 +
.../tajo/worker/AbstractResourceAllocator.java | 14 +-
.../apache/tajo/worker/ResourceAllocator.java | 6 +-
.../tajo/worker/TajoResourceAllocator.java | 101 +++---
.../tajo/worker/TajoWorkerClientService.java | 16 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 16 +-
.../apache/tajo/worker/TaskRunnerHistory.java | 14 +-
.../src/main/proto/ContainerProtocol.proto | 48 +++
.../src/main/proto/QueryMasterProtocol.proto | 3 +
.../main/proto/ResourceTrackerProtocol.proto | 3 +
.../src/main/proto/TajoMasterProtocol.proto | 7 +-
.../src/main/proto/TajoWorkerProtocol.proto | 5 +-
.../src/main/resources/webapps/admin/index.jsp | 2 +-
.../resources/webapps/admin/query_executor.jsp | 4 +-
.../org/apache/tajo/cluster/TestServerName.java | 102 ------
.../apache/tajo/engine/query/TestCTASQuery.java | 1 +
.../tajo/engine/query/TestHBaseTable.java | 9 +-
.../tajo/engine/query/TestInsertQuery.java | 14 +-
.../tajo/engine/query/TestTablePartitions.java | 56 ++++
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 181 ++++++++---
.../tajo/master/rm/TestTajoResourceManager.java | 16 +-
.../dataset/TestTajoJdbc/table1/table1.tbl | 5 +
.../TestTajoJdbc/create_table_with_date_ddl.sql | 10 +
...rtitionedTableWithSmallerExpressions5.result | 7 +
...rtitionedTableWithSmallerExpressions6.result | 4 +
.../TestTajoJdbc/testSortWithDateTime.result | 7 +
.../src/main/sphinx/hcatalog_integration.rst | 13 +
.../org/apache/tajo/jdbc/MetaDataTuple.java | 4 +-
.../main/java/org/apache/tajo/plan/Target.java | 2 +-
.../org/apache/tajo/plan/expr/CaseWhenEval.java | 2 +-
.../org/apache/tajo/plan/expr/EvalNode.java | 2 +-
.../tajo/plan/function/GeneralFunction.java | 2 +-
.../apache/tajo/plan/logical/BinaryNode.java | 2 +-
.../apache/tajo/plan/logical/LogicalNode.java | 2 +-
.../tajo/plan/serder/EvalNodeAdapter.java | 4 +-
.../tajo/plan/serder/LogicalNodeAdapter.java | 4 +-
.../apache/tajo/plan/serder/PlanGsonHelper.java | 2 +-
.../apache/tajo/rpc/RemoteCallException.java | 6 +-
.../org/apache/tajo/storage/StorageManager.java | 237 ++++++++------
.../src/main/resources/storage-default.xml | 16 +-
tajo-storage/tajo-storage-hdfs/pom.xml | 7 +-
.../apache/tajo/storage/FileStorageManager.java | 78 +++--
.../tajo/storage/json/JsonLineDeserializer.java | 220 +++++++++++++
.../apache/tajo/storage/json/JsonLineSerDe.java | 37 +++
.../tajo/storage/json/JsonLineSerializer.java | 130 ++++++++
.../tajo/storage/text/ByteBufLineReader.java | 22 +-
.../tajo/storage/text/CSVLineDeserializer.java | 96 ++++++
.../apache/tajo/storage/text/CSVLineSerDe.java | 41 +++
.../tajo/storage/text/CSVLineSerializer.java | 70 +++++
.../tajo/storage/text/DelimitedTextFile.java | 231 +++++++-------
.../text/TextFieldSerializerDeserializer.java | 2 +-
.../tajo/storage/text/TextLineDeserializer.java | 60 ++++
.../tajo/storage/text/TextLineParsingError.java | 31 ++
.../apache/tajo/storage/text/TextLineSerDe.java | 65 ++++
.../tajo/storage/text/TextLineSerializer.java | 45 +++
.../apache/tajo/storage/TestFileSystems.java | 266 ++++++++--------
.../org/apache/tajo/storage/TestStorages.java | 107 ++++---
.../apache/tajo/storage/avro/TestAvroUtil.java | 2 +-
.../tajo/storage/json/TestLineReader.java | 197 ++++++++++++
.../java/org/apache/tajo/storage/s3/INode.java | 124 --------
.../storage/s3/InMemoryFileSystemStore.java | 176 -----------
.../apache/tajo/storage/s3/S3OutputStream.java | 234 --------------
.../tajo/storage/s3/SmallBlockS3FileSystem.java | 314 -------------------
.../dataset/TestJsonSerDe/testVariousType.json | 1 +
.../src/test/resources/dataset/testLineText.txt | 2 +
.../resources/dataset/testVariousTypes.avsc | 20 ++
.../src/test/resources/storage-default.xml | 16 +-
.../src/test/resources/testVariousTypes.avsc | 21 --
159 files changed, 3760 insertions(+), 2310 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 87ef9a6,188e024..6769e9f
--- a/CHANGES
+++ b/CHANGES
@@@ -15,10 -11,16 +15,19 @@@ Release 0.9.1 - unrelease
TAJO-235: Support Oracle CatalogStore. (Jihun Kang via hyunsik)
+ TAJO-1095: Implement Json file scanner. (hyunsik)
+
IMPROVEMENT
+ TAJO-1122: Refactor the tajo-storage project structure.
+ (Hyoungjun Kim)
++
+ TAJO-1165: Needs to show error messages on query_executor.jsp.
+ (Jihun Kang via jaehwa)
+
+ TAJO-1204: Remove unused ServerName class. (DaeMyung Kang via jaehwa)
+
+ TAJO-1053: ADD PARTITIONS for HCatalogStore. (jaehwa)
TAJO-1195: Remove unused CachedDNSResolver Class. (DaeMyung Kang via jaehwa)
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
----------------------------------------------------------------------
diff --cc tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
index 4056ed3,4056ed3..0216128
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
@@@ -22,7 -22,7 +22,7 @@@ import com.google.common.base.Objects
import com.google.gson.*;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import org.apache.tajo.util.TUtil;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-algebra/src/main/java/org/apache/tajo/algebra/Expr.java
----------------------------------------------------------------------
diff --cc tajo-algebra/src/main/java/org/apache/tajo/algebra/Expr.java
index b53a6db,b53a6db..6a41caf
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Expr.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Expr.java
@@@ -22,7 -22,7 +22,7 @@@ import com.google.gson.*
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
import org.apache.tajo.algebra.LiteralValue.LiteralType;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import java.lang.reflect.Type;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java
index 69d5be4,69d5be4..3a3eb66
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java
@@@ -23,7 -23,7 +23,7 @@@ import com.google.gson.annotations.Expo
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTableDescProto;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index a6e1a31,f2d9b9c..aca2460
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@@ -278,8 -280,8 +280,10 @@@ public class CatalogUtil
return StoreType.AVRO;
} else if (typeStr.equalsIgnoreCase(StoreType.TEXTFILE.name())) {
return StoreType.TEXTFILE;
+ } else if (typeStr.equalsIgnoreCase(StoreType.JSON.name())) {
+ return StoreType.JSON;
+ } else if (typeStr.equalsIgnoreCase(StoreType.HBASE.name())) {
+ return StoreType.HBASE;
} else {
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
index aceb6f1,aceb6f1..f6acfae
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java
@@@ -25,7 -25,7 +25,7 @@@ import org.apache.tajo.catalog.proto.Ca
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.DataType;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
/**
* Describes a column. It is an immutable object.
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
index 23d39f2,23d39f2..5aea768
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
@@@ -22,7 -22,7 +22,7 @@@ import com.google.common.base.Objects
import com.google.gson.annotations.Expose;
import org.apache.tajo.annotation.NotNull;
import org.apache.tajo.function.*;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos.FunctionDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 296230b,296230b..eae52e5
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@@ -25,13 -25,13 +25,12 @@@ import org.apache.commons.logging.Log
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.exception.AlreadyExistsFieldException;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
--import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.util.TUtil;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java
index 5b10dfd,5b10dfd..3390146
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java
@@@ -22,7 -22,7 +22,7 @@@ import com.google.common.base.Objects
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.common.ProtoObject;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import static org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index ce167e1,ce167e1..001b71a
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@@ -30,7 -30,7 +30,7 @@@ import org.apache.tajo.catalog.proto.Ca
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.ProtoObject;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
index 2d95e6b,2d95e6b..6378d42
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
@@@ -28,7 -28,7 +28,7 @@@ import org.apache.tajo.catalog.proto.Ca
import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableProtoOrBuilder;
import org.apache.tajo.common.ProtoObject;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.util.KeyValueSet;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
index 2ecbe98,2ecbe98..5090a7d
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
@@@ -25,7 -25,7 +25,7 @@@ import org.apache.tajo.catalog.TableMet
import org.apache.tajo.function.Function;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.Datum;
--import org.apache.tajo.json.*;
++import org.apache.tajo.storage.json.*;
import org.apache.tajo.util.TUtil;
import java.lang.reflect.Type;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/FunctionAdapter.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/FunctionAdapter.java
index 14722c3,14722c3..849d970
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/FunctionAdapter.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/FunctionAdapter.java
@@@ -20,8 -20,8 +20,8 @@@ package org.apache.tajo.catalog.json
import com.google.gson.*;
import org.apache.tajo.function.Function;
--import org.apache.tajo.json.CommonGsonHelper;
--import org.apache.tajo.json.GsonSerDerAdapter;
++import org.apache.tajo.storage.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.GsonSerDerAdapter;
import java.lang.reflect.Type;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
index 3f4304a,3f4304a..33d504e
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
@@@ -20,11 -20,11 +20,11 @@@ package org.apache.tajo.catalog.json
import com.google.common.base.Preconditions;
import com.google.gson.*;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
--import org.apache.tajo.json.GsonSerDerAdapter;
++import org.apache.tajo.storage.json.GsonSerDerAdapter;
import java.lang.reflect.Type;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
index d775ba8,d775ba8..9d520d4
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java
@@@ -23,7 -23,7 +23,7 @@@ import com.google.gson.annotations.Expo
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
/**
* <code>PartitionDesc</code> presents a table partition.
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java
index 721a7a0,721a7a0..adc5620
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java
@@@ -26,7 -26,7 +26,7 @@@ import org.apache.tajo.catalog.Schema
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.util.TUtil;
import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
index 1bd0276,1bd0276..ef6e35a
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
@@@ -30,7 -30,7 +30,7 @@@ import org.apache.tajo.catalog.proto.Ca
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.util.TUtil;
public class ColumnStats implements ProtoObject<CatalogProtos.ColumnStatsProto>, Cloneable, GsonObject {
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
index dd358ae,dd358ae..9df01da
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
@@@ -26,7 -26,7 +26,7 @@@ import com.google.gson.Gson
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
import org.apache.tajo.common.ProtoObject;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java
index 6a538b8,6a538b8..8842746
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java
@@@ -21,7 -21,7 +21,7 @@@ package org.apache.tajo.function
import com.google.common.base.Objects;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.proto.CatalogProtos;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.GsonObject;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.datum.Datum;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 92437af,f29bc6c..26951dd
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@@ -36,7 -36,7 +36,8 @@@ enum StoreType
SEQUENCEFILE = 8;
AVRO = 9;
TEXTFILE = 10;
- HBASE = 11;
+ JSON = 11;
++ HBASE = 12;
}
enum OrderType {
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
index 442db71,442db71..dcf966c
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java
@@@ -23,8 -23,8 +23,8 @@@ import org.apache.tajo.conf.TajoConf
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.exception.InvalidCastException;
import org.apache.tajo.exception.InvalidOperationException;
--import org.apache.tajo.json.CommonGsonHelper;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.GsonObject;
import static org.apache.tajo.common.TajoDataTypes.Type;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java
index c7ec7ae,c7ec7ae..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/ClassNameSerializer.java
+++ /dev/null
@@@ -1,47 -1,47 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--/**
-- *
-- */
--package org.apache.tajo.json;
--
--import com.google.gson.*;
--import org.apache.tajo.util.ClassUtil;
--
--import java.lang.reflect.Type;
--
--public class ClassNameSerializer implements GsonSerDerAdapter<Class> {
--
-- @Override
-- public JsonElement serialize(Class clazz, Type type,
-- JsonSerializationContext ctx) {
-- return new JsonPrimitive(clazz.getName());
-- }
--
-- @Override
-- public Class deserialize(JsonElement json, Type type,
-- JsonDeserializationContext ctx) throws JsonParseException {
-- try {
-- return ClassUtil.forName(json.getAsString());
-- } catch (ClassNotFoundException e) {
-- e.printStackTrace();
-- }
-- return null;
-- }
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java
index 8994b11,8994b11..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java
+++ /dev/null
@@@ -1,92 -1,92 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--package org.apache.tajo.json;
--
--import com.google.gson.Gson;
--import com.google.gson.GsonBuilder;
--import com.google.gson.JsonElement;
--import com.google.gson.JsonNull;
--import com.google.gson.JsonObject;
--import com.google.gson.JsonParseException;
--import org.apache.tajo.datum.Datum;
--import org.apache.tajo.util.TUtil;
--
--import java.lang.reflect.Type;
--import java.util.Map;
--
--public class CommonGsonHelper {
-- private static Gson gson;
-- private static Gson gsonPretty;
--
-- private CommonGsonHelper() {
-- }
--
-- private static Map<Type, GsonSerDerAdapter> registerAdapters() {
-- Map<Type, GsonSerDerAdapter> adapters = TUtil.newHashMap();
-- adapters.put(Datum.class, new DatumAdapter());
--
-- return adapters;
-- }
--
-- public static Gson getInstance() {
-- if (gson == null ) {
-- GsonHelper helper = new GsonHelper(registerAdapters());
-- gson = helper.getGson();
-- }
-- return gson;
-- }
--
-- public static Gson getPrettyInstance() {
-- if (gsonPretty == null) {
-- GsonBuilder prettyBuilder = new GsonBuilder()
-- .setPrettyPrinting()
-- .excludeFieldsWithoutExposeAnnotation();
-- GsonHelper.registerAdapters(prettyBuilder, registerAdapters());
-- gsonPretty = prettyBuilder.create();
-- }
--
-- return gsonPretty;
-- }
--
-- public static String toJson(GsonObject object, Class<? extends GsonObject> clazz) {
-- return getInstance().toJson(object, clazz);
-- }
--
-- public static <T extends GsonObject> T fromJson(String json, Class<T> clazz) {
-- return getInstance().fromJson(json, clazz);
-- }
--
-- /**
-- * A helper method that gets a JSON object member value after making sure it exists and has a valid value. Useful when
-- * a member value should present to proceed.
-- * @param object A JSON object to get a member value from
-- * @param memberName The name of a member to get value of
-- * @return {@link JsonElement} value read from the given member
-- * @throws JsonParseException When the specified member does not exist or have a value.
-- */
-- public static JsonElement getOrDie(JsonObject object, String memberName) throws JsonParseException {
-- if (object.has(memberName)) {
-- JsonElement element = object.get(memberName);
-- if (!JsonNull.INSTANCE.equals(element)) {
-- return element;
-- }
-- }
-- throw new JsonParseException("Field '" + memberName + "' not found in JSON object '" + object + "'");
-- }
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/DataTypeAdapter.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/DataTypeAdapter.java
index fc15aea,fc15aea..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/DataTypeAdapter.java
+++ /dev/null
@@@ -1,63 -1,63 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--package org.apache.tajo.json;
--
--import com.google.gson.*;
--import org.apache.tajo.common.TajoDataTypes;
--import org.apache.tajo.common.TajoDataTypes.DataType;
--
--import java.lang.reflect.Type;
--
--
--public class DataTypeAdapter implements GsonSerDerAdapter<DataType> {
--
-- @Override
-- public DataType deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
-- throws JsonParseException {
--
-- JsonObject obj = (JsonObject) json;
-- DataType.Builder builder = DataType.newBuilder();
-- TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(CommonGsonHelper.getOrDie(obj, "type").getAsString());
-- builder.setType(type);
--
-- JsonElement len = obj.get("len");
-- if (len != null) {
-- builder.setLength(len.getAsInt());
-- }
-- JsonElement code = obj.get("code");
-- if (code != null) {
-- builder.setCode(code.getAsString());
-- }
-- return builder.build();
-- }
--
-- @Override
-- public JsonElement serialize(DataType src, Type typeOfSrc, JsonSerializationContext context) {
-- JsonObject json = new JsonObject();
-- json.addProperty("type", src.getType().name());
-- if (src.hasLength()) {
-- json.addProperty("len", src.getLength());
-- }
-- if (src.hasCode()) {
-- json.addProperty("code", src.getCode());
-- }
--
-- return json;
-- }
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java
index d65559d,d65559d..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/DatumAdapter.java
+++ /dev/null
@@@ -1,76 -1,76 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--package org.apache.tajo.json;
--
--import com.google.gson.*;
--import org.apache.tajo.common.TajoDataTypes;
--import org.apache.tajo.datum.*;
--
--import java.lang.reflect.Type;
--
--public class DatumAdapter implements GsonSerDerAdapter<Datum> {
--
-- @Override
-- public Datum deserialize(JsonElement json, Type typeOfT,
-- JsonDeserializationContext context) throws JsonParseException {
-- JsonObject jsonObject = json.getAsJsonObject();
-- String typeName = CommonGsonHelper.getOrDie(jsonObject, "type").getAsString();
-- TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(typeName);
-- switch (type) {
-- case DATE:
-- return new DateDatum(CommonGsonHelper.getOrDie(jsonObject, "value").getAsInt());
-- case TIME:
-- return new TimeDatum(CommonGsonHelper.getOrDie(jsonObject, "value").getAsLong());
-- case TIMESTAMP:
-- return new TimestampDatum(CommonGsonHelper.getOrDie(jsonObject, "value").getAsLong());
-- case INTERVAL:
-- String[] values = CommonGsonHelper.getOrDie(jsonObject, "value").getAsString().split(",");
--
-- return new IntervalDatum(Integer.parseInt(values[0]), Long.parseLong(values[1]));
-- default:
-- return context.deserialize(CommonGsonHelper.getOrDie(jsonObject, "body"),
-- DatumFactory.getDatumClass(TajoDataTypes.Type.valueOf(typeName)));
-- }
-- }
--
-- @Override
-- public JsonElement serialize(Datum src, Type typeOfSrc, JsonSerializationContext context) {
-- JsonObject jsonObj = new JsonObject();
-- jsonObj.addProperty("type", src.type().name());
-- switch (src.type()) {
-- case DATE:
-- jsonObj.addProperty("value", src.asInt4());
-- break;
-- case TIME:
-- jsonObj.addProperty("value", src.asInt8());
-- break;
-- case TIMESTAMP:
-- jsonObj.addProperty("value", src.asInt8());
-- break;
-- case INTERVAL:
-- IntervalDatum interval = (IntervalDatum)src;
-- jsonObj.addProperty("value", interval.getMonths() + "," + interval.getMilliSeconds());
-- break;
-- default:
-- jsonObj.add("body", context.serialize(src));
-- }
--
-- return jsonObj;
-- }
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/GsonHelper.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/GsonHelper.java
index 8c6d891,8c6d891..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/GsonHelper.java
+++ /dev/null
@@@ -1,49 -1,49 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--package org.apache.tajo.json;
--
--import com.google.gson.*;
--
--import java.lang.reflect.Type;
--import java.util.Map;
--
--public class GsonHelper {
-- private final GsonBuilder builder;
-- private final Gson gson;
--
-- public GsonHelper(Map<Type, GsonSerDerAdapter> adapters) {
-- builder = new GsonBuilder().excludeFieldsWithoutExposeAnnotation();
-- registerAdapters(builder, adapters);
-- gson = builder.create();
-- }
--
-- public static void registerAdapters(GsonBuilder builder, Map<Type, GsonSerDerAdapter> adapters) {
-- for (Map.Entry<Type, GsonSerDerAdapter> entry : adapters.entrySet()) {
-- try {
-- builder.registerTypeAdapter(entry.getKey(), entry.getValue());
-- } catch (Exception e) {
-- e.printStackTrace();
-- }
-- }
-- }
--
-- public Gson getGson() {
-- return gson;
-- }
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/GsonObject.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/GsonObject.java
index 3c49a21,3c49a21..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/GsonObject.java
+++ /dev/null
@@@ -1,23 -1,23 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--package org.apache.tajo.json;
--
--public interface GsonObject {
-- public String toJson();
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/GsonSerDerAdapter.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/GsonSerDerAdapter.java
index 80a40f8,80a40f8..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/GsonSerDerAdapter.java
+++ /dev/null
@@@ -1,25 -1,25 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--package org.apache.tajo.json;
--
--import com.google.gson.JsonDeserializer;
--import com.google.gson.JsonSerializer;
--
--public interface GsonSerDerAdapter<T> extends JsonSerializer<T>, JsonDeserializer<T> {
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/json/PathSerializer.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/json/PathSerializer.java
index c286ae3,c286ae3..0000000
deleted file mode 100644,100644
--- a/tajo-common/src/main/java/org/apache/tajo/json/PathSerializer.java
+++ /dev/null
@@@ -1,41 -1,41 +1,0 @@@
--/**
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--
--/**
-- *
-- */
--package org.apache.tajo.json;
--
--import com.google.gson.*;
--import org.apache.hadoop.fs.Path;
--
--import java.lang.reflect.Type;
--
--public class PathSerializer implements GsonSerDerAdapter<Path> {
--
-- @Override
-- public JsonElement serialize(Path path, Type arg1,
-- JsonSerializationContext arg2) {
-- return new JsonPrimitive(path.toString());
-- }
--
-- @Override
-- public Path deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext context) throws JsonParseException {
-- return new Path(arg0.getAsJsonPrimitive().getAsString());
-- }
--}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
index 11ac9b7,a3d8de0..942718b
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
@@@ -32,7 -32,8 +32,20 @@@ public class StorageConstants
public static final String TEXT_DELIMITER = "text.delimiter";
public static final String TEXT_NULL = "text.null";
- public static final String TEXTFILE_SERDE = "textfile.serde";
+ public static final String TEXT_SERDE_CLASS = "text.serde.class";
+ public static final String DEFAULT_TEXT_SERDE_CLASS = "org.apache.tajo.storage.text.CSVLineSerDe";
++ /**
++ * It's the maximum number of parsing error torrence.
++ *
++ * <ul>
++ * <li>If it is -1, it is always torrent against any parsing error.</li>
++ * <li>If it is 0, it does not permit any parsing error.</li>
++ * <li>If it is some positive integer (i.e., > 0), the given number of parsing errors in each
++ * task will be permissible</li>
++ * </ul>
++ **/
++ public static final String TEXT_ERROR_TOLERANCE_MAXNUM = "text.error-tolerance.max-num";
++ public static final String DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM = "0";
@Deprecated
public static final String SEQUENCEFILE_DELIMITER = "sequencefile.delimiter";
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/ClassNameSerializer.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/ClassNameSerializer.java
index 0000000,0000000..7069134
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/ClassNameSerializer.java
@@@ -1,0 -1,0 +1,47 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++/**
++ *
++ */
++package org.apache.tajo.storage.json;
++
++import com.google.gson.*;
++import org.apache.tajo.util.ClassUtil;
++
++import java.lang.reflect.Type;
++
++public class ClassNameSerializer implements GsonSerDerAdapter<Class> {
++
++ @Override
++ public JsonElement serialize(Class clazz, Type type,
++ JsonSerializationContext ctx) {
++ return new JsonPrimitive(clazz.getName());
++ }
++
++ @Override
++ public Class deserialize(JsonElement json, Type type,
++ JsonDeserializationContext ctx) throws JsonParseException {
++ try {
++ return ClassUtil.forName(json.getAsString());
++ } catch (ClassNotFoundException e) {
++ e.printStackTrace();
++ }
++ return null;
++ }
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/CommonGsonHelper.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/CommonGsonHelper.java
index 0000000,0000000..e2d20cf
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/CommonGsonHelper.java
@@@ -1,0 -1,0 +1,92 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.tajo.storage.json;
++
++import com.google.gson.Gson;
++import com.google.gson.GsonBuilder;
++import com.google.gson.JsonElement;
++import com.google.gson.JsonNull;
++import com.google.gson.JsonObject;
++import com.google.gson.JsonParseException;
++import org.apache.tajo.datum.Datum;
++import org.apache.tajo.util.TUtil;
++
++import java.lang.reflect.Type;
++import java.util.Map;
++
++public class CommonGsonHelper {
++ private static Gson gson;
++ private static Gson gsonPretty;
++
++ private CommonGsonHelper() {
++ }
++
++ private static Map<Type, GsonSerDerAdapter> registerAdapters() {
++ Map<Type, GsonSerDerAdapter> adapters = TUtil.newHashMap();
++ adapters.put(Datum.class, new DatumAdapter());
++
++ return adapters;
++ }
++
++ public static Gson getInstance() {
++ if (gson == null ) {
++ GsonHelper helper = new GsonHelper(registerAdapters());
++ gson = helper.getGson();
++ }
++ return gson;
++ }
++
++ public static Gson getPrettyInstance() {
++ if (gsonPretty == null) {
++ GsonBuilder prettyBuilder = new GsonBuilder()
++ .setPrettyPrinting()
++ .excludeFieldsWithoutExposeAnnotation();
++ GsonHelper.registerAdapters(prettyBuilder, registerAdapters());
++ gsonPretty = prettyBuilder.create();
++ }
++
++ return gsonPretty;
++ }
++
++ public static String toJson(GsonObject object, Class<? extends GsonObject> clazz) {
++ return getInstance().toJson(object, clazz);
++ }
++
++ public static <T extends GsonObject> T fromJson(String json, Class<T> clazz) {
++ return getInstance().fromJson(json, clazz);
++ }
++
++ /**
++ * A helper method that gets a JSON object member value after making sure it exists and has a valid value. Useful when
++ * a member value should present to proceed.
++ * @param object A JSON object to get a member value from
++ * @param memberName The name of a member to get value of
++ * @return {@link JsonElement} value read from the given member
++ * @throws JsonParseException When the specified member does not exist or have a value.
++ */
++ public static JsonElement getOrDie(JsonObject object, String memberName) throws JsonParseException {
++ if (object.has(memberName)) {
++ JsonElement element = object.get(memberName);
++ if (!JsonNull.INSTANCE.equals(element)) {
++ return element;
++ }
++ }
++ throw new JsonParseException("Field '" + memberName + "' not found in JSON object '" + object + "'");
++ }
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/DataTypeAdapter.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/DataTypeAdapter.java
index 0000000,0000000..e8765c9
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/DataTypeAdapter.java
@@@ -1,0 -1,0 +1,63 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.tajo.storage.json;
++
++import com.google.gson.*;
++import org.apache.tajo.common.TajoDataTypes;
++import org.apache.tajo.common.TajoDataTypes.DataType;
++
++import java.lang.reflect.Type;
++
++
++public class DataTypeAdapter implements GsonSerDerAdapter<DataType> {
++
++ @Override
++ public DataType deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
++ throws JsonParseException {
++
++ JsonObject obj = (JsonObject) json;
++ DataType.Builder builder = DataType.newBuilder();
++ TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(CommonGsonHelper.getOrDie(obj, "type").getAsString());
++ builder.setType(type);
++
++ JsonElement len = obj.get("len");
++ if (len != null) {
++ builder.setLength(len.getAsInt());
++ }
++ JsonElement code = obj.get("code");
++ if (code != null) {
++ builder.setCode(code.getAsString());
++ }
++ return builder.build();
++ }
++
++ @Override
++ public JsonElement serialize(DataType src, Type typeOfSrc, JsonSerializationContext context) {
++ JsonObject json = new JsonObject();
++ json.addProperty("type", src.getType().name());
++ if (src.hasLength()) {
++ json.addProperty("len", src.getLength());
++ }
++ if (src.hasCode()) {
++ json.addProperty("code", src.getCode());
++ }
++
++ return json;
++ }
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/DatumAdapter.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/DatumAdapter.java
index 0000000,0000000..b176720
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/DatumAdapter.java
@@@ -1,0 -1,0 +1,76 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.tajo.storage.json;
++
++import com.google.gson.*;
++import org.apache.tajo.common.TajoDataTypes;
++import org.apache.tajo.datum.*;
++
++import java.lang.reflect.Type;
++
++public class DatumAdapter implements GsonSerDerAdapter<Datum> {
++
++ @Override
++ public Datum deserialize(JsonElement json, Type typeOfT,
++ JsonDeserializationContext context) throws JsonParseException {
++ JsonObject jsonObject = json.getAsJsonObject();
++ String typeName = CommonGsonHelper.getOrDie(jsonObject, "type").getAsString();
++ TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(typeName);
++ switch (type) {
++ case DATE:
++ return new DateDatum(CommonGsonHelper.getOrDie(jsonObject, "value").getAsInt());
++ case TIME:
++ return new TimeDatum(CommonGsonHelper.getOrDie(jsonObject, "value").getAsLong());
++ case TIMESTAMP:
++ return new TimestampDatum(CommonGsonHelper.getOrDie(jsonObject, "value").getAsLong());
++ case INTERVAL:
++ String[] values = CommonGsonHelper.getOrDie(jsonObject, "value").getAsString().split(",");
++
++ return new IntervalDatum(Integer.parseInt(values[0]), Long.parseLong(values[1]));
++ default:
++ return context.deserialize(CommonGsonHelper.getOrDie(jsonObject, "body"),
++ DatumFactory.getDatumClass(TajoDataTypes.Type.valueOf(typeName)));
++ }
++ }
++
++ @Override
++ public JsonElement serialize(Datum src, Type typeOfSrc, JsonSerializationContext context) {
++ JsonObject jsonObj = new JsonObject();
++ jsonObj.addProperty("type", src.type().name());
++ switch (src.type()) {
++ case DATE:
++ jsonObj.addProperty("value", src.asInt4());
++ break;
++ case TIME:
++ jsonObj.addProperty("value", src.asInt8());
++ break;
++ case TIMESTAMP:
++ jsonObj.addProperty("value", src.asInt8());
++ break;
++ case INTERVAL:
++ IntervalDatum interval = (IntervalDatum)src;
++ jsonObj.addProperty("value", interval.getMonths() + "," + interval.getMilliSeconds());
++ break;
++ default:
++ jsonObj.add("body", context.serialize(src));
++ }
++
++ return jsonObj;
++ }
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/GsonHelper.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/GsonHelper.java
index 0000000,0000000..2e193ba
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/GsonHelper.java
@@@ -1,0 -1,0 +1,49 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.tajo.storage.json;
++
++import com.google.gson.*;
++
++import java.lang.reflect.Type;
++import java.util.Map;
++
++public class GsonHelper {
++ private final GsonBuilder builder;
++ private final Gson gson;
++
++ public GsonHelper(Map<Type, GsonSerDerAdapter> adapters) {
++ builder = new GsonBuilder().excludeFieldsWithoutExposeAnnotation();
++ registerAdapters(builder, adapters);
++ gson = builder.create();
++ }
++
++ public static void registerAdapters(GsonBuilder builder, Map<Type, GsonSerDerAdapter> adapters) {
++ for (Map.Entry<Type, GsonSerDerAdapter> entry : adapters.entrySet()) {
++ try {
++ builder.registerTypeAdapter(entry.getKey(), entry.getValue());
++ } catch (Exception e) {
++ e.printStackTrace();
++ }
++ }
++ }
++
++ public Gson getGson() {
++ return gson;
++ }
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/GsonObject.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/GsonObject.java
index 0000000,0000000..aa4a82d
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/GsonObject.java
@@@ -1,0 -1,0 +1,23 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.tajo.storage.json;
++
++public interface GsonObject {
++ public String toJson();
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/GsonSerDerAdapter.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/GsonSerDerAdapter.java
index 0000000,0000000..493a997
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/GsonSerDerAdapter.java
@@@ -1,0 -1,0 +1,25 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.tajo.storage.json;
++
++import com.google.gson.JsonDeserializer;
++import com.google.gson.JsonSerializer;
++
++public interface GsonSerDerAdapter<T> extends JsonSerializer<T>, JsonDeserializer<T> {
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/storage/json/PathSerializer.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/storage/json/PathSerializer.java
index 0000000,0000000..6d9d32f
new file mode 100644
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/json/PathSerializer.java
@@@ -1,0 -1,0 +1,41 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++/**
++ *
++ */
++package org.apache.tajo.storage.json;
++
++import com.google.gson.*;
++import org.apache.hadoop.fs.Path;
++
++import java.lang.reflect.Type;
++
++public class PathSerializer implements GsonSerDerAdapter<Path> {
++
++ @Override
++ public JsonElement serialize(Path path, Type arg1,
++ JsonSerializationContext arg2) {
++ return new JsonPrimitive(path.toString());
++ }
++
++ @Override
++ public Path deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext context) throws JsonParseException {
++ return new Path(arg0.getAsJsonPrimitive().getAsString());
++ }
++}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
index 8e3eb2a,8e3eb2a..a52651b
--- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
@@@ -22,8 -22,8 +22,8 @@@ import com.google.common.base.Objects
import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
import org.apache.tajo.common.ProtoObject;
--import org.apache.tajo.json.CommonGsonHelper;
--import org.apache.tajo.json.GsonObject;
++import org.apache.tajo.storage.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.GsonObject;
import java.util.HashMap;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java
----------------------------------------------------------------------
diff --cc tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java
index 4dcbbee,4dcbbee..d42b496
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java
@@@ -19,7 -19,7 +19,7 @@@
package org.apache.tajo.datum;
import org.apache.tajo.common.TajoDataTypes.Type;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import org.apache.tajo.util.Bytes;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
----------------------------------------------------------------------
diff --cc tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
index 41b4dca,41b4dca..56673f2
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java
@@@ -20,7 -20,7 +20,7 @@@ package org.apache.tajo.datum
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.InvalidCastException;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import org.junit.Test;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java
----------------------------------------------------------------------
diff --cc tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java
index 14ebc7c,14ebc7c..0ef905d
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java
@@@ -18,7 -18,7 +18,7 @@@
package org.apache.tajo.datum;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java
----------------------------------------------------------------------
diff --cc tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java
index ea641ec,ea641ec..a130c0b
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java
@@@ -21,7 -21,7 +21,7 @@@ package org.apache.tajo.datum
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.exception.InvalidCastException;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
----------------------------------------------------------------------
diff --cc tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
index 5f27cfa,5886083..f4b5c97
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
@@@ -21,7 -21,7 +21,7 @@@ package org.apache.tajo.datum
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.InvalidCastException;
--import org.apache.tajo.json.CommonGsonHelper;
++import org.apache.tajo.storage.json.CommonGsonHelper;
import org.apache.tajo.util.datetime.DateTimeUtil;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
index 57f2536,57f2536..1d10e67
--- a/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
@@@ -31,7 -31,7 +31,7 @@@ import org.apache.tajo.plan.function.Ag
import org.apache.tajo.plan.function.GeneralFunction;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.function.Function;
--import org.apache.tajo.json.*;
++import org.apache.tajo.storage.json.*;
import org.apache.tajo.plan.serder.EvalNodeAdapter;
import org.apache.tajo.plan.serder.LogicalNodeAdapter;
import org.apache.tajo.util.TUtil;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
index f38ae3c,a63b838..247b373
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
@@@ -117,13 -122,33 +119,31 @@@ public class PhysicalPlanUtil
return FragmentConvertor.toFragmentProtoArray(fragments.toArray(new FileFragment[]{}));
}
+ /**
+ *
+ * @param fs
+ * @param path The table path
+ * @param result The final result files to be used
+ * @param startFileIndex
+ * @param numResultFiles
+ * @param currentFileIndex
+ * @param partitioned A flag to indicate if this table is partitioned
+ * @param currentDepth Current visiting depth of partition directories
+ * @param maxDepth The partition depth of this table
+ * @throws IOException
+ */
private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List<FileStatus> result,
int startFileIndex, int numResultFiles,
- AtomicInteger currentFileIndex) throws IOException {
+ AtomicInteger currentFileIndex, boolean partitioned,
+ int currentDepth, int maxDepth) throws IOException {
+ // Intermediate directory
if (fs.isDirectory(path)) {
-
- FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter);
-
+ FileStatus[] files = fs.listStatus(path, FileStorageManager.hiddenFileFilter);
if (files != null && files.length > 0) {
+
for (FileStatus eachFile : files) {
+
+ // checking if the enough number of files are found
if (result.size() >= numResultFiles) {
return;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
index b20c5d8,77e3257..01137aa
--- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
@@@ -41,9 -41,9 +41,10 @@@ import org.apache.tajo.master.event.Tas
import org.apache.tajo.master.querymaster.QueryUnit;
import org.apache.tajo.master.querymaster.QueryUnitAttempt;
import org.apache.tajo.master.querymaster.SubQuery;
+ import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.storage.DataLocation;
import org.apache.tajo.storage.fragment.FileFragment;
+import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.worker.FetchImpl;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
index 038c334,b2883cc..c691a98
--- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
@@@ -40,10 -37,8 +39,10 @@@ import org.apache.tajo.master.event.Tas
import org.apache.tajo.master.querymaster.QueryUnit;
import org.apache.tajo.master.querymaster.QueryUnitAttempt;
import org.apache.tajo.master.querymaster.SubQuery;
+ import org.apache.tajo.master.container.TajoContainerId;
+import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.storage.fragment.FileFragment;
- import org.apache.tajo.storage.StorageManager;
+import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.worker.FetchImpl;
http://git-wip-us.apache.org/repos/asf/tajo/blob/940546a0/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------