You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2013/05/09 07:38:35 UTC
[03/11] TAJO-57: Recognize Parser and Catalog Standard SQL data
types. (hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFile.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFile.java
deleted file mode 100644
index 83df49e..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFile.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
-import tajo.catalog.proto.CatalogProtos.CompressType;
-import tajo.datum.Datum;
-import tajo.storage.exception.UnknownCodecException;
-import tajo.storage.exception.UnknownDataTypeException;
-import tajo.storage.hcfile.reader.Reader;
-import tajo.storage.hcfile.writer.Writer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class HCFile {
-
- public static class Scanner extends ColumnFileScanner {
- private final Log LOG = LogFactory.getLog(Scanner.class);
- public static final int HDFS_CHUNK_SIZE = 65536;
- private HColumnMetaWritable meta;
- private FileSystem fs;
- private FSDataInputStream in;
- private ByteBuffer buffer;
- private UpdatableSeekableBlock block;
- private Reader reader;
- private Index<Long> index;
- private int nextBlock;
-
- public Scanner(Configuration conf, Path path)
- throws IOException, UnknownDataTypeException, UnknownCodecException {
- super(conf, path);
- fs = path.getFileSystem(conf);
- in = fs.open(path);
- buffer = ByteBuffer.allocateDirect(HDFS_CHUNK_SIZE);
- buffer.flip();
- index = new Index<Long>();
- FileStatus file = fs.getFileStatus(path);
-
- readColumnHeader(file.getLen());
- if (this.meta.isCompressed()) {
- reader = new CompressedBlockReader(this.meta.getDataType(), this.meta.getCompressType());
- } else {
- reader = new BlockReader(this.meta.getDataType());
- }
- }
-
- public Path getPath() {
- return this.path;
- }
-
- protected boolean fillBuffer(int desired) throws IOException {
- buffer.compact();
- int read = in.read(buffer);
- if (read == -1) {
- return false;
- } else {
- if (read < desired) {
- read += in.read(buffer);
- if (read < desired) {
- return false;
- }
- }
- buffer.flip();
- return true;
- }
- }
-
- private void readColumnHeader(long fileLen) throws IOException {
- in.seek(fileLen - HColumnMetaWritable.size());
- long headerPos = in.getPos();
- meta = new HColumnMetaWritable();
- meta.readFields(in);
-
- // read index
- in.seek(meta.getOffsetToIndex());
- while (in.getPos() < headerPos) {
- index.add(new IndexItem<Long>(in.readLong(), in.readLong()));
- }
-
- in.seek(0);
- nextBlock = 0;
- block = null;
- }
-
-// private void readColumnHeader(long fileLen) throws IOException {
-// in.seek(fileLen - Long.SIZE / 8);
-// long headerPos = in.readLong();
-// in.seek(headerPos);
-//
-// // read header
-// long headerLen = fileLen - Long.SIZE/8 - headerPos;
-// byte[] buf = new byte[(int)headerLen];
-// in.read(buf);
-// ColumnMetaProto proto = ColumnMetaProto.parseFrom(buf);
-// meta = new HCFileMetaImpl(proto);
-//
-// // read index
-// in.seek(((HCFileMetaImpl)meta).getOffsetToIndex());
-// while (in.getPos() < headerPos) {
-// index.add(new IndexItem(in.readLong(), in.readLong()));
-// }
-//
-// in.seek(0);
-// currentBlock = 0;
-// block = null;
-// }
-
- @Override
- public ColumnMeta getMeta() throws IOException {
- return this.meta;
- }
-
- @Override
- public void first() throws IOException {
- if (nextBlock != 1) {
- pos(0);
- }
- }
-
- @Override
- public void last() throws IOException {
- if (nextBlock != index.size()) {
- IndexItem<Long> item = index.get(index.size()-1);
- long lastOffset = item.getValue();
- in.seek(lastOffset);
- nextBlock = index.size()-1;
- block = null;
- buffer.clear();
- buffer.flip();
- getBlock();
- block.last();
- }
- }
-
- @Override
- public void pos(long rid) throws IOException {
- // Consider the case which the given rid is already involved in the block
- if (block == null
- || block.getMeta().getStartRid() > rid
- || block.getMeta().getStartRid()+block.getMeta().getRecordNum() <= rid) {
- IndexItem<Long> item = index.searchLargestSmallerThan(rid);
- long offset = item.getValue();
- in.seek(offset);
- nextBlock = index.findPos(item);
- block = null;
- // block을 읽을 경우 buffer를 완전히 비워야 함
- buffer.clear();
- buffer.flip();
- getBlock();
- }
-
- block.pos(rid-block.getMeta().getStartRid());
- }
-
- @Override
- public Datum get() throws IOException {
- Datum ret;
- if (block == null || (ret=block.next()) == null) {
- getBlock();
- if (block == null) {
- return null;
- }
- ret = block.next();
- }
- return ret;
- }
-
- @Override
- public Block getBlock() throws IOException {
- if (nextBlock < index.size()) {
- if (buffer.remaining() < Integer.SIZE/8) {
- if (!fillBuffer(Integer.SIZE/8-buffer.remaining())) {
- return null;
- }
- }
- int blockSize = buffer.getInt();
- if (buffer.remaining() < blockSize) {
- if (!fillBuffer(blockSize-buffer.remaining())) {
- return null;
- }
- }
-
- int originLimit = buffer.limit();
- buffer.limit(buffer.position() + blockSize);
- BlockMeta blockMeta = newBlockMeta();
-
- block = (UpdatableSeekableBlock) reader.read(blockMeta, buffer);
- buffer.limit(originLimit);
- nextBlock++;
- return block;
- } else {
- return null;
- }
- }
-
- private BlockMeta newBlockMeta() throws IOException {
- return new HBlockMetaImpl(
- meta.getDataType(),
- 0, // not set
- index.get(nextBlock).getRid(),
- meta.isSorted(),
- meta.isContiguous(),
- meta.isCompressed());
- }
-
- @Override
- public Datum[] getBlockAsArray() throws IOException {
- return this.getBlock().asArray();
- }
-
- @Override
- public long getPos() throws IOException {
- // TODO
- return block.getMeta().getStartRid() + block.getPos();
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- in.close();
- }
- }
-
- public static class Appender extends ColumnFileAppender {
- private Log LOG = LogFactory.getLog(Appender.class);
- private final int HDFS_BLOCK_SIZE;
- private FSDataOutputStream out;
- private FileSystem fs;
- private UpdatableBlock block;
- private BlockMeta blockMeta;
- private long blockStartId;
- private long startId;
- private int blockRecordNum;
- private int totalRecordNum;
- private Writer writer;
- private Index<Long> index;
-
- public Appender(Configuration conf, ColumnMeta meta, Path path)
- throws IOException, UnknownDataTypeException, UnknownCodecException {
- super(conf, meta, path);
- HDFS_BLOCK_SIZE = conf.getInt("dfs.blocksize", -1);
- if (HDFS_BLOCK_SIZE == -1) {
- throw new IOException("HDFS block size can not be initialized!");
- } else {
- LOG.info("HDFS block size: " + HDFS_BLOCK_SIZE);
- }
- fs = path.getFileSystem(conf);
- out = fs.create(path);
-
- if (this.meta.getCompressType() != CompressType.COMP_NONE) {
- writer = new CompressedBlockWriter(out,
- this.meta.getDataType(), this.meta.getCompressType());
- } else {
- writer = new BlockWriter(out, this.meta.getDataType());
- blockMeta = new HBlockMetaImpl(this.meta.getDataType(), 0, 0,
- this.meta.isSorted(), this.meta.isContiguous(), this.meta.isCompressed());
- block = new BasicBlock();
- }
- index = new Index<Long>();
- blockStartId = ((HColumnMetaWritable)this.meta).getStartRid();
- startId = blockStartId;
- blockRecordNum = 0;
- totalRecordNum = 0;
- }
-
- public Path getPath() {
- return this.path;
- }
-
- public boolean isAppendable(Datum datum) throws IOException {
- // TODO: validation of (index.size()+1)
- int indexSize = (index.size()+1) * 2 * Long.SIZE/8;
- if (!block.isAppendable(datum)) {
- indexSize += 2 * Long.SIZE/8;
- }
-
- return out.getPos() /* written size */
- + block.getSize()
- + Integer.SIZE/8 /* block size */
- + ColumnStoreUtil.getWrittenSize(datum)
- + HColumnMetaWritable.size()
- + Long.SIZE/8 /* column header pos */
- + indexSize
- < HDFS_BLOCK_SIZE;
- }
-
- public long getStartId() {
- return this.startId;
- }
-
- public int getRecordNum() {
- return this.totalRecordNum;
- }
-
- @Override
- public void append(Datum datum) throws IOException {
- if (!block.isAppendable(datum)) {
- flush();
- blockRecordNum = 0;
- }
- block.appendValue(datum);
- blockRecordNum++;
- totalRecordNum++;
- }
-
- @Override
- public void flush() throws IOException {
- index.add(new IndexItem(blockStartId, writer.getPos()));
- blockMeta.setStartRid(blockStartId)
- .setRecordNum(blockRecordNum);
- block.setMeta(blockMeta);
- writer.write(block);
- block.clear();
- blockStartId += blockRecordNum;
- }
-
- private void writeColumnHeader() throws IOException {
- long offsetToIndex = out.getPos();
- // write index
- for (IndexItem<Long> e : index.get()) {
- out.writeLong(e.getRid());
- out.writeLong(e.getValue());
- }
-
- // write header
- HColumnMetaWritable columnMeta = (HColumnMetaWritable) meta;
- columnMeta.setRecordNum(totalRecordNum);
- columnMeta.setOffsetToIndex((int) offsetToIndex);
- columnMeta.write(out);
-
-// long headerPos = out.getPos();
-//
-// HCFileMetaImpl hmeta = (HCFileMetaImpl) meta;
-// hmeta.setRecordNum(totalRecordNum);
-// hmeta.setOffsetToIndex((int)offsetToIndex);
-// ColumnMetaProto proto = meta.getProto();
-// FileUtil.writeProto(out, proto);
-//
-// out.writeLong(headerPos);
- }
-
- @Override
- public void close() throws IOException {
- if (blockRecordNum > 0) {
- flush();
- }
- writeColumnHeader();
-
- writer.close();
- out.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFileMetaImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFileMetaImpl.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFileMetaImpl.java
deleted file mode 100644
index e06e944..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCFileMetaImpl.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import tajo.catalog.proto.CatalogProtos.*;
-import tajo.catalog.proto.CatalogProtos.ColumnMetaProto.Builder;
-import tajo.common.ProtoObject;
-
-public class HCFileMetaImpl implements ColumnMeta, ProtoObject<ColumnMetaProto> {
- private ColumnMetaProto proto = ColumnMetaProto.getDefaultInstance();
- private Builder builder = null;
- private boolean viaProto = false;
-
- private Long startRid;
- private Integer recordNum;
- private Integer offsetToIndex;
- private DataType dataType;
- private StoreType storeType;
- private CompressType compType;
- private Boolean compressed;
- private Boolean sorted;
- private Boolean contiguous;
-
- public HCFileMetaImpl() {
- setModified();
- storeType = StoreType.HCFILE;
- }
-
- public HCFileMetaImpl(long startRid, DataType dataType,
- CompressType compType, boolean compressed,
- boolean sorted, boolean contiguous) {
- this();
- this.startRid = startRid;
- this.dataType = dataType;
- this.compType = compType;
- this.compressed = compressed;
- this.sorted = sorted;
- this.contiguous = contiguous;
- }
-
- public HCFileMetaImpl(ColumnMetaProto proto) {
- this.proto = proto;
- this.viaProto = true;
- }
-
- public void setStartRid(Long startRid) {
- setModified();
- this.startRid = startRid;
- }
-
- public void setRecordNum(Integer recordNum) {
- setModified();
- this.recordNum = recordNum;
- }
-
- public void setOffsetToIndex(Integer offsetToIndex) {
- setModified();
- this.offsetToIndex = offsetToIndex;
- }
-
- public void setDataType(DataType dataType) {
- setModified();
- this.dataType = dataType;
- }
-
- public void setCompressed(Boolean compressed) {
- setModified();
- this.compressed = compressed;
- }
-
- public void setSorted(Boolean sorted) {
- setModified();
- this.sorted = sorted;
- }
-
- public void setContiguous(Boolean contiguous) {
- setModified();
- this.contiguous = contiguous;
- }
-
- public void setCompType(CompressType compType) {
- setModified();
- this.compType = compType;
- }
-
- public Long getStartRid() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (startRid != null) {
- return this.startRid;
- }
- if (!p.hasStartRid()) {
- return null;
- }
- this.startRid = p.getStartRid();
-
- return startRid;
- }
-
- public Integer getRecordNum() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (recordNum != null) {
- return this.recordNum;
- }
- if (!p.hasRecordNum()) {
- return null;
- }
- this.recordNum = p.getRecordNum();
- return recordNum;
- }
-
- public Integer getOffsetToIndex() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (offsetToIndex != null) {
- return this.offsetToIndex;
- }
- if (!p.hasOffsetToIndex()) {
- return null;
- }
- this.offsetToIndex = p.getOffsetToIndex();
- return offsetToIndex;
- }
-
- @Override
- public StoreType getStoreType() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (storeType != null) {
- return this.storeType;
- }
- if (!p.hasStoreType()) {
- return null;
- }
- storeType = p.getStoreType();
- return this.storeType;
- }
-
- @Override
- public DataType getDataType() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (dataType != null) {
- return dataType;
- }
- if (!p.hasDataType()) {
- return null;
- }
- dataType = p.getDataType();
- return dataType;
- }
-
- @Override
- public CompressType getCompressType() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (compType != null) {
- return compType;
- }
- if (!p.hasCompType()) {
- return null;
- }
- compType = p.getCompType();
- return compType;
- }
-
- @Override
- public boolean isCompressed() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (compressed != null) {
- return compressed;
- }
- if (!p.hasCompressed()) {
- return false;
- }
-
- compressed = p.getCompressed();
- return compressed;
- }
-
- @Override
- public boolean isSorted() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (sorted != null) {
- return sorted;
- }
- if (!p.hasSorted()) {
- return false;
- }
-
- sorted = p.getSorted();
- return sorted;
- }
-
- @Override
- public boolean isContiguous() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (contiguous != null) {
- return contiguous;
- }
- if (!p.hasContiguous()) {
- return false;
- }
- contiguous = p.getContiguous();
- return contiguous;
- }
-
- @Override
- public void initFromProto() {
- ColumnMetaProtoOrBuilder p = viaProto ? proto : builder;
- if (this.startRid == null && p.hasStartRid()) {
- this.startRid = p.getStartRid();
- }
- if (this.recordNum == null && p.hasRecordNum()) {
- this.recordNum = p.getRecordNum();
- }
- if (this.offsetToIndex == null && p.hasOffsetToIndex()) {
- this.offsetToIndex = p.getOffsetToIndex();
- }
- if (this.dataType == null && p.hasDataType()) {
- this.dataType = p.getDataType();
- }
- if (this.compressed == null && p.hasCompressed()) {
- this.compressed = p.getCompressed();
- }
- if (this.sorted == null && p.hasSorted()) {
- this.sorted = p.getSorted();
- }
- if (this.contiguous == null && p.hasContiguous()) {
- this.contiguous = p.getContiguous();
- }
- if (this.storeType == null && p.hasStoreType()) {
- this.storeType = p.getStoreType();
- }
- if (this.compType == null && p.hasCompType()) {
- this.compType = p.getCompType();
- }
- }
-
- @Override
- public ColumnMetaProto getProto() {
- if(!viaProto) {
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- return proto;
- }
-
- private void setModified() {
- if (viaProto && builder == null) {
- builder = ColumnMetaProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private void mergeLocalToBuilder() {
- if (builder == null) {
- builder = ColumnMetaProto.newBuilder(proto);
- }
- if (this.startRid != null) {
- builder.setStartRid(startRid);
- }
- if (this.recordNum != null) {
- builder.setRecordNum(recordNum);
- }
- if (this.offsetToIndex != null) {
- builder.setOffsetToIndex(offsetToIndex);
- }
- if (this.dataType != null) {
- builder.setDataType(dataType);
- }
- if (this.compressed != null) {
- builder.setCompressed(compressed);
- }
- if (this.sorted != null) {
- builder.setSorted(sorted);
- }
- if (this.contiguous != null) {
- builder.setContiguous(contiguous);
- }
- if (this.storeType != null) {
- builder.setStoreType(storeType);
- }
- if (this.compType != null) {
- builder.setCompType(compType);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCTupleAppender.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCTupleAppender.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCTupleAppender.java
deleted file mode 100644
index 5de66a5..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HCTupleAppender.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import tajo.catalog.Column;
-import tajo.catalog.Schema;
-import tajo.catalog.TableMeta;
-import tajo.catalog.proto.CatalogProtos.CompressType;
-import tajo.catalog.proto.CatalogProtos.DataType;
-import tajo.storage.Tuple;
-import tajo.storage.exception.UnknownCodecException;
-import tajo.storage.exception.UnknownDataTypeException;
-import tajo.storage.hcfile.HCFile.Appender;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class HCTupleAppender implements TupleAppender {
- private final Log LOG = LogFactory.getLog(HCTupleAppender.class);
- private Configuration conf;
- private Schema schema;
- private final Path dataDir;
- private int shardId;
- private Map<Column, Appender> columnAppenderMap = Maps.newHashMap();
- private Map<Column, Integer> columnFileIdMap = Maps.newHashMap();
- private Column baseColumn;
- private int baseColumnIdx;
-
- public HCTupleAppender(Configuration conf, TableMeta meta, int baseColumnIdx, Path tablePath)
- throws IOException, UnknownCodecException, UnknownDataTypeException {
- this.conf = conf;
- this.schema = meta.getSchema();
- this.baseColumnIdx = baseColumnIdx;
- this.baseColumn = schema.getColumn(baseColumnIdx);
- this.dataDir = new Path(tablePath, "data");
- this.shardId = -1;
- newShard();
- }
-
- @Override
- public void addTuple(Tuple t) throws IOException {
- Appender appender = columnAppenderMap.get(baseColumn);
-
- // try base column
- if (!appender.isAppendable(t.get(baseColumnIdx))) {
- try {
- newShard();
- appender = columnAppenderMap.get(baseColumn);
- } catch (UnknownDataTypeException e) {
- throw new IOException(e);
- } catch (UnknownCodecException e) {
- throw new IOException(e);
- }
- }
- appender.append(t.get(baseColumnIdx));
-
- for (int i = 0; i < schema.getColumnNum(); i++) {
- if (i == baseColumnIdx) continue;
-
- appender = columnAppenderMap.get(schema.getColumn(i));
- if (!appender.isAppendable(t.get(i))) {
- try {
- appender = newAppender(schema.getColumn(i));
- } catch (UnknownDataTypeException e) {
- LOG.info(e);
- } catch (UnknownCodecException e) {
- LOG.info(e);
- }
- }
- appender.append(t.get(i));
- }
- }
-
- private Appender newAppender(Column column)
- throws UnknownCodecException, IOException, UnknownDataTypeException {
- long nextStartId;
-
- if (columnAppenderMap.containsKey(column)) {
- Appender oldAppender = columnAppenderMap.get(column);
- nextStartId = oldAppender.getStartId() + oldAppender.getRecordNum();
- oldAppender.close();
- } else {
- nextStartId = 0;
- }
-
- ColumnMeta columnMeta = newColumnMeta(nextStartId, column.getDataType());
- Path columnPath = new Path(dataDir, getColumnFileName(column));
- LOG.info("new appender is initialized for " + column.getColumnName());
- LOG.info("column path: " + columnPath.toString());
- Appender newAppender = new Appender(conf, columnMeta, columnPath);
- columnAppenderMap.put(column, newAppender);
- return newAppender;
- }
-
- private ColumnMeta newColumnMeta(long startId, DataType dataType) {
- return new HColumnMetaWritable(startId, dataType, CompressType.COMP_NONE,
- false, false, true);
- }
-
- private String getColumnFileName(Column column) {
- int fileId;
- if (columnFileIdMap.containsKey(column)) {
- fileId = columnFileIdMap.get(column) + 1;
- } else {
- fileId = 0;
- }
- columnFileIdMap.put(column, fileId);
- return shardId + "/" + column.getColumnName() + "_" + fileId;
- }
-
- private void newShard()
- throws UnknownDataTypeException, IOException, UnknownCodecException {
- ++shardId;
- columnFileIdMap.clear();
- LOG.info("new shard id: " + shardId);
- long oldStartRid = -1, newStartRid = -1;
-
- if (!columnAppenderMap.isEmpty()) {
- oldStartRid = columnAppenderMap.get(schema.getColumn(0)).getStartId();
- }
-
- for (Column column : schema.getColumns()) {
- newAppender(column);
- }
-
- if (oldStartRid != -1) {
- newStartRid = columnAppenderMap.get(schema.getColumn(0)).getStartId();
- writeMeta(shardId-1, oldStartRid, newStartRid-oldStartRid);
- }
-
-// index.add(new IndexItem(columnAppenderMap.get(schema.getColumn(0)).getStartId(), shardId));
- }
-
- @Override
- public void flush() throws IOException {
- for (Appender appender : columnAppenderMap.values()) {
- appender.flush();
- }
- }
-
- @Override
- public void close() throws IOException {
- Appender app = columnAppenderMap.get(schema.getColumn(0));
- writeMeta(shardId, app.getStartId(), app.getRecordNum());
- for (Appender appender : columnAppenderMap.values()) {
- appender.close();
- }
-// writeIndex();
- }
-
- private void writeMeta(int shardId, long startRid, long length) throws IOException {
- FileSystem fs = dataDir.getFileSystem(conf);
- Path shardPath = new Path(dataDir, shardId+"");
- Path metaPath = new Path(shardPath, ".smeta");
- FSDataOutputStream out = fs.create(metaPath);
- out.writeLong(startRid);
- out.writeLong(length);
- out.close();
- }
-
-/* private void writeIndex() throws IOException {
- FileSystem fs = dataDir.getFileSystem(conf);
- Path indexPath = new Path(dataDir, ".index");
- FSDataOutputStream out = fs.create(indexPath);
- for (IndexItem<Integer> item : index.get()) {
- out.writeLong(item.getRid());
- out.writeInt(item.getValue());
- }
- out.close();
- }*/
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnMetaWritable.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnMetaWritable.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnMetaWritable.java
deleted file mode 100644
index 4a49f9e..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnMetaWritable.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import org.apache.hadoop.io.Writable;
-import tajo.catalog.proto.CatalogProtos.CompressType;
-import tajo.catalog.proto.CatalogProtos.DataType;
-import tajo.catalog.proto.CatalogProtos.StoreType;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-public class HColumnMetaWritable implements ColumnMeta, Writable {
- public static final int SIZE = Long.SIZE/8
- + 2 * Integer.SIZE/8
- + 2 * Short.SIZE/8
- + 3 * Byte.SIZE/8;
-
-
- private long startRid;
- private int recordNum;
- private int offsetToIndex;
- private DataType dataType;
- private CompressType compType;
- private boolean compressed;
- private boolean sorted;
- private boolean contiguous;
-
- public HColumnMetaWritable() {
-
- }
-
- public HColumnMetaWritable(long startRid, DataType dataType,
- CompressType compType, boolean compressed,
- boolean sorted, boolean contiguous) {
- this.startRid = startRid;
- this.dataType = dataType;
- this.compType = compType;
- this.compressed = compressed;
- this.sorted = sorted;
- this.contiguous = contiguous;
- }
-
- public void setStartRid(long startRid) {
- this.startRid = startRid;
- }
-
- public void setRecordNum(int recordNum) {
- this.recordNum = recordNum;
- }
-
- public void setOffsetToIndex(int offsetToIndex) {
- this.offsetToIndex = offsetToIndex;
- }
-
- public void setDataType(DataType dataType) {
- this.dataType = dataType;
- }
-
- public void setCompType(CompressType compType) {
- this.compType = compType;
- }
-
- public void setCompressed(boolean compressed) {
- this.compressed = compressed;
- }
-
- public void setSorted(boolean sorted) {
- this.sorted = sorted;
- }
-
- public void setContiguous(boolean contiguous) {
- this.contiguous = contiguous;
- }
-
- @Override
- public StoreType getStoreType() {
- return StoreType.HCFILE;
- }
-
- @Override
- public DataType getDataType() {
- return dataType;
- }
-
- @Override
- public CompressType getCompressType() {
- return compType;
- }
-
- @Override
- public boolean isCompressed() {
- return compressed;
- }
-
- @Override
- public boolean isSorted() {
- return sorted;
- }
-
- @Override
- public boolean isContiguous() {
- return contiguous;
- }
-
- public long getStartRid() {
- return startRid;
- }
-
- public int getRecordNum() {
- return recordNum;
- }
-
- public long getOffsetToIndex() {
- return this.offsetToIndex;
-
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeLong(startRid);
- out.writeInt(recordNum);
- out.writeInt(offsetToIndex);
- out.writeShort(dataType.getNumber());
- out.writeShort(compType.getNumber());
- out.writeBoolean(compressed);
- out.writeBoolean(sorted);
- out.writeBoolean(contiguous);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- startRid = in.readLong();
- recordNum = in.readInt();
- offsetToIndex = in.readInt();
- dataType = DataType.valueOf(in.readShort());
- compType = CompressType.valueOf(in.readShort());
- compressed = in.readBoolean();
- sorted = in.readBoolean();
- contiguous = in.readBoolean();
- }
-
- public static int size() {
- return SIZE;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnReader.java
deleted file mode 100644
index 9994c68..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/HColumnReader.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import tajo.catalog.Column;
-import tajo.catalog.Schema;
-import tajo.catalog.TableMeta;
-import tajo.catalog.TableMetaImpl;
-import tajo.catalog.proto.CatalogProtos.TableProto;
-import tajo.datum.Datum;
-import tajo.storage.exception.UnknownCodecException;
-import tajo.storage.exception.UnknownDataTypeException;
-import tajo.storage.hcfile.HCFile.Scanner;
-import tajo.util.FileUtil;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-public class HColumnReader implements ColumnReader {
- private final Log LOG = LogFactory.getLog(HColumnReader.class);
- private final Configuration conf;
- private final FileSystem fs;
- private Scanner scanner;
- private Path dataDir;
- private List<Path> dataPaths = Lists.newArrayList();
- private int next;
- private Index<Integer> index;
- private Column target;
-
- public HColumnReader(Configuration conf, Path tablePath, int targetColumnIdx)
- throws IOException {
- this.conf = conf;
- this.fs = tablePath.getFileSystem(this.conf);
- Schema schema = getSchema(tablePath);
- init(conf, tablePath, schema.getColumn(targetColumnIdx));
- }
-
- public HColumnReader(Configuration conf, Path tablePath, String columnName)
- throws IOException {
- this.conf = conf;
- this.fs = tablePath.getFileSystem(this.conf);
- Schema schema = getSchema(tablePath);
- init(conf, tablePath, schema.getColumnByName(columnName));
- }
-
- public HColumnReader(Configuration conf, Path tablePath, Column target)
- throws IOException {
- this.conf = conf;
- this.fs = tablePath.getFileSystem(this.conf);
- init(conf, tablePath, target);
- }
-
- public List<Path> getDataPaths() {
- return this.dataPaths;
- }
-
- private void init(Configuration conf, Path tablePath, Column target) throws IOException {
- this.dataDir = new Path(tablePath, "data");
- FileStatus[] files = fs.listStatus(dataDir);
- Path[] shardPaths = new Path[files.length];
- int i, j = 0;
- for (i = 0; i < files.length; i++) {
- shardPaths[j++] = files[i].getPath();
- }
- Arrays.sort(shardPaths);
-// next = 0;
- index = new Index<Integer>();
- this.target = target;
- initDataPaths(shardPaths);
- }
-
- private void setTarget(Path tablePath, int targetColumnIdx) throws IOException {
- Schema schema = getSchema(tablePath);
- this.target = schema.getColumn(targetColumnIdx);
- }
-
- private void setTarget(Path tablePath, String columnName) throws IOException {
- Schema schema = getSchema(tablePath);
- this.target = schema.getColumn(columnName);
- }
-
- private Schema getSchema(Path tablePath) throws IOException {
- Path metaPath = new Path(tablePath, ".meta");
- TableProto proto = (TableProto) FileUtil.loadProto(fs, metaPath,
- TableProto.getDefaultInstance());
- TableMeta meta = new TableMetaImpl(proto);
- return meta.getSchema();
- }
-
- private void initDataPaths(Path[] shardPaths) throws IOException {
- Path indexPath;
- FSDataInputStream in;
- long rid;
- String targetName = target.getColumnName();
-
- for (int i = 0; i < shardPaths.length; i++) {
- indexPath = new Path(shardPaths[i], ".smeta");
- in = fs.open(indexPath);
-
- rid = in.readLong();
- index.add(new IndexItem(rid, i));
-
- in.close();
-
- FileStatus[] columnFiles = fs.listStatus(shardPaths[i]);
- for (FileStatus file : columnFiles) {
- String colName = file.getPath().getName();
- if (colName.length() < targetName.length()) {
- continue;
- }
- if (colName.substring(0, targetName.length()).equals(targetName)) {
- LOG.info("column file: " + file.getPath().toString());
- dataPaths.add(file.getPath());
- }
- }
- }
- next = 0;
- }
-
- @Override
- public void first() throws IOException {
- pos(0);
- }
-
- @Override
- public void last() throws IOException {
- // TODO
- }
-
- @Override
- public Datum get() throws IOException {
- Datum datum;
-
- if (scanner == null) {
- if (next < dataPaths.size()) {
- scanner = getScanner(conf, dataPaths.get(next++));
- return scanner.get();
- } else {
- return null;
- }
- } else {
- datum = scanner.get();
- if (datum == null) {
- scanner.close();
- scanner = null;
- if (next < dataPaths.size()) {
- scanner = getScanner(conf, dataPaths.get(next++));
- return scanner.get();
- } else {
- return null;
- }
- }
- return datum;
- }
- }
-
- @Override
- public void pos(long rid) throws IOException {
- if (scanner != null) {
- HColumnMetaWritable meta = (HColumnMetaWritable) scanner.getMeta();
- if (meta.getStartRid() <= rid
- && meta.getStartRid() + meta.getRecordNum() > rid) {
- scanner.pos(rid);
- return;
- } else {
- scanner.close();
- scanner = null;
- }
- }
-
- // scanner must be null
- IndexItem<Integer> item = index.searchLargestSmallerThan(rid);
- if (item == null) {
- throw new IOException("Cannot find the column file containing " + rid);
- }
- Path shardPath = new Path(dataDir, item.getValue().toString());
-
- int i;
- FileStatus[] files = fs.listStatus(shardPath);
- for (FileStatus file : files) {
- if (dataPaths.contains(file.getPath())) {
- LOG.info("matched path: " + file.getPath());
- scanner = getScanner(conf, file.getPath());
- HColumnMetaWritable meta = (HColumnMetaWritable) scanner.getMeta();
- LOG.info("start: " + meta.getStartRid() + " len: " + meta.getRecordNum());
- if (meta.getStartRid() <= rid
- && meta.getStartRid() + meta.getRecordNum() > rid) {
- scanner.pos(rid);
-
- for (i = 0; i < dataPaths.size(); i++) {
- if (file.getPath().equals(dataPaths.get(i))) {
- next = i+1;
- break;
- }
- }
- if (i == dataPaths.size()) {
- throw new IOException("Invalid path: " + file.getPath().toString());
- }
- return;
- }
- scanner.close();
- scanner = null;
- }
- }
-
- if (scanner == null) {
- throw new IOException("Null scanner");
- }
- }
-
- private Scanner getScanner(Configuration conf, Path columnPath) throws IOException {
- try {
- scanner = new Scanner(conf, columnPath);
- return scanner;
- } catch (UnknownDataTypeException e) {
- throw new IOException(e);
- } catch (UnknownCodecException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public long getPos() throws IOException {
- // TODO
- return scanner.getPos();
- }
-
- @Override
- public void close() throws IOException {
- if (scanner != null) {
- scanner.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Index.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Index.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Index.java
deleted file mode 100644
index 7e6dbf8..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Index.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import com.google.common.collect.Lists;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class Index<T> {
- private List<IndexItem<T>> index = Lists.newArrayList();
-
- public Index() {
-
- }
-
- public void add(IndexItem<T> item) {
- index.add(item);
- }
-
- public void sort() {
- IndexItem<T>[] array = index.toArray(new IndexItem[index.size()]);
- index.clear();
- Arrays.sort(array);
- for (IndexItem<T> item : array) {
- index.add(item);
- }
- }
-
- public IndexItem<T> get(int i) {
- return index.get(i);
- }
-
- public void clear() {
- index.clear();
- }
-
- public int findPos(IndexItem<T> searchItem) {
- for (int i = 0; i < index.size(); i++) {
- if (index.get(i).equals(searchItem)) {
- return i;
- }
- }
- return -1;
- }
-
- public int size() {
- return index.size();
- }
-
- public List<IndexItem<T>> get() {
- return index;
- }
-
- /**
- * For a given rid, search the index item
- * @param rid search rid
- * @return If the rid is in the index, return the index item which has the rid.
- * Else, return null
- */
- public IndexItem<T> searchExact(long rid) {
- if (index.isEmpty()) {
- return null;
- }
-
- return searchExact(rid, 0, index.size());
- }
-
- private IndexItem<T> searchExact(long rid, int start, int length) {
- int leftLength = length/2;
- IndexItem<T> mid = index.get(start + leftLength);
- if (mid.getRid() == rid) {
- return mid;
- } else if (length == 1) {
- return null;
- }
-
- if (mid.getRid() > rid) {
- return searchExact(rid, start, leftLength);
- } else if (mid.getRid() < rid) {
- return searchExact(rid, start + leftLength, length - leftLength);
- }
- return null;
- }
-
- /**
- * For a given rid,
- * search the index item of which rid is the largest smaller than the given rid.
- *
- * @param rid search rid
- * @return If the given rid is in the index, return the index item which has the rid.
- * Else, return the index item which has the largest rid smaller than the given rid.
- */
- public IndexItem<T> searchLargestSmallerThan(long rid) {
- if (index.isEmpty()) {
- return null;
- }
- return searchLargestSmallerThan(rid, 0, index.size());
- }
-
- private IndexItem<T> searchLargestSmallerThan(long rid, int start, int length) {
- int leftLength = length/2;
- IndexItem<T> mid = index.get(start + leftLength);
- if (mid.getRid() == rid) {
- return mid;
- } else if (length == 1) {
- return index.get(start+leftLength);
- }
-
- if (mid.getRid() > rid) {
- return searchLargestSmallerThan(rid, start, leftLength);
- } else if (mid.getRid() < rid) {
- return searchLargestSmallerThan(rid, start + leftLength, length - leftLength);
- }
-
- return null;
- }
-
- /**
- * For a given rid,
- * search the index item of which rid is the smallest larger than the given rid.
- *
- * @param rid search rid
- * @return If the given rid is in the index, return the index item which has the rid.
- * Else, return the index item which has the smallest rid larger than the given rid.
- */
- public IndexItem<T> searchSmallestLargerThan(long rid) {
- if (index.isEmpty()) {
- return null;
- }
- return searchSmallestLargerThan(rid, 0, index.size());
- }
-
- private IndexItem<T> searchSmallestLargerThan(long rid, int start, int length) {
- int leftLength = length/2;
- IndexItem<T> mid = index.get(start + leftLength);
- if (mid.getRid() == rid) {
- return mid;
- } else if (length == 1) {
- if (start+leftLength+1 >= index.size()) {
- return null;
- } else {
- return index.get(start+leftLength+1);
- }
- }
-
- if (mid.getRid() > rid) {
- return searchSmallestLargerThan(rid, start, leftLength);
- } else if (mid.getRid() < rid) {
- return searchSmallestLargerThan(rid, start + leftLength, length - leftLength);
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/IndexItem.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/IndexItem.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/IndexItem.java
deleted file mode 100644
index 8ce62c3..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/IndexItem.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-public class IndexItem<T> implements Comparable<IndexItem> {
- private long rid;
- private T value;
-
- public IndexItem() {
-
- }
-
- public IndexItem(long rid, T value) {
- this.set(rid, value);
- }
-
- public void set(long rid, T offset) {
- this.rid = rid;
- this.value = offset;
- }
-
- public T getValue() {
- return value;
- }
-
- public long getRid() {
- return rid;
- }
-
- @Override
- public int compareTo(IndexItem indexItem) {
- return (int)(this.rid - indexItem.rid);
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof IndexItem) {
- IndexItem item = (IndexItem) o;
- if (this.rid == item.rid &&
- this.value.equals(item.value)) {
- return true;
- }
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Seekable.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Seekable.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Seekable.java
deleted file mode 100644
index 1a82270..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Seekable.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import java.io.IOException;
-
-public interface Seekable {
-
- public void first() throws IOException;
-
- public void last() throws IOException;
-
- public void pos(long rid) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/TupleAppender.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/TupleAppender.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/TupleAppender.java
deleted file mode 100644
index bf19a50..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/TupleAppender.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import tajo.storage.Tuple;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-public interface TupleAppender extends Closeable {
-
- public void addTuple(Tuple t) throws IOException;
- public void flush() throws IOException;
- public void close() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Updatable.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Updatable.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Updatable.java
deleted file mode 100644
index 5952d3d..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/Updatable.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-import tajo.datum.Datum;
-
-import java.io.IOException;
-
-public interface Updatable {
-
- void appendValue(Datum data) throws IOException;
-
- void setValues(Datum[] data) throws IOException;
-
- Datum removeValue() throws IOException;
-
- void clear() throws IOException;
-
- boolean isAppendable(Datum datum) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableBlock.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableBlock.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableBlock.java
deleted file mode 100644
index 8321075..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableBlock.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-public abstract class UpdatableBlock implements Updatable, Block {
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableSeekableBlock.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableSeekableBlock.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableSeekableBlock.java
deleted file mode 100644
index b2d6ea6..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/UpdatableSeekableBlock.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile;
-
-public abstract class UpdatableSeekableBlock extends UpdatableBlock implements Seekable {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/BitVector.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/BitVector.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/BitVector.java
deleted file mode 100644
index 15944dd..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/BitVector.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.compress;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class BitVector extends Codec {
- @Override
- public byte[] decompress(byte[] compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] decompress(ByteBuffer compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(byte[] decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(ByteBuffer decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Codec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Codec.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Codec.java
deleted file mode 100644
index 2fe019b..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Codec.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.compress;
-
-import tajo.catalog.proto.CatalogProtos.CompressType;
-import tajo.storage.exception.UnknownCodecException;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public abstract class Codec {
-
- public static Codec get(CompressType type) throws UnknownCodecException {
- switch (type) {
- case COMP_NONE:
- return null;
- case NULL_SUPPRESS:
- return new NullSuppress();
- case RUN_LENGTH:
- return new RunLength();
- case BIT_VECTOR:
- return new BitVector();
- case DICTIONARY:
- return new Dictionary();
- case SNAPPY:
- return new Snappy();
- case LZ:
- return new LZ();
- default:
- throw new UnknownCodecException(type.name());
- }
- }
-
- public abstract byte[] decompress(byte[] compressed) throws IOException;
- public abstract byte[] decompress(ByteBuffer compressed) throws IOException;
-
- public abstract byte[] compress(byte[] decompressed) throws IOException;
- public abstract byte[] compress(ByteBuffer decompressed) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Dictionary.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Dictionary.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Dictionary.java
deleted file mode 100644
index 681b855..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Dictionary.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.compress;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class Dictionary extends Codec {
- @Override
- public byte[] decompress(byte[] compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] decompress(ByteBuffer compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(byte[] decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(ByteBuffer decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/LZ.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/LZ.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/LZ.java
deleted file mode 100644
index ad1f9c8..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/LZ.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.compress;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class LZ extends Codec {
- @Override
- public byte[] decompress(byte[] compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] decompress(ByteBuffer compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(byte[] decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(ByteBuffer decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/NullSuppress.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/NullSuppress.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/NullSuppress.java
deleted file mode 100644
index 18dc09f..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/NullSuppress.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.compress;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class NullSuppress extends Codec {
- @Override
- public byte[] decompress(byte[] compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] decompress(ByteBuffer compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(byte[] decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(ByteBuffer decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/RunLength.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/RunLength.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/RunLength.java
deleted file mode 100644
index 9402181..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/RunLength.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.compress;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class RunLength extends Codec {
- @Override
- public byte[] decompress(byte[] compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] decompress(ByteBuffer compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(byte[] decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(ByteBuffer decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Snappy.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Snappy.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Snappy.java
deleted file mode 100644
index 33cd64b..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/compress/Snappy.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.compress;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class Snappy extends Codec {
- @Override
- public byte[] decompress(byte[] compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] decompress(ByteBuffer compressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(byte[] decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public byte[] compress(ByteBuffer decompressed) throws IOException {
- return new byte[0]; //To change body of implemented methods use File | Settings | File Templates.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ArrayReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ArrayReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ArrayReader.java
deleted file mode 100644
index a7215b0..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ArrayReader.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.ArrayDatum;
-import tajo.datum.Datum;
-import tajo.datum.json.GsonCreator;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class ArrayReader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- int arrayByteSize = buffer.getInt();
- byte [] arrayBytes = new byte[arrayByteSize];
- buffer.get(arrayBytes);
- String json = new String(arrayBytes);
- ArrayDatum array = (ArrayDatum) GsonCreator
- .getInstance().fromJson(json, Datum.class);
- return array;
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ByteReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ByteReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ByteReader.java
deleted file mode 100644
index 3a37fd7..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/ByteReader.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.Datum;
-import tajo.datum.DatumFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class ByteReader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- return DatumFactory.createByte(buffer.get());
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/BytesReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/BytesReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/BytesReader.java
deleted file mode 100644
index 6a22904..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/BytesReader.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.Datum;
-import tajo.datum.DatumFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class BytesReader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- int size = buffer.getInt();
- byte[] rawBytes = new byte[size];
- buffer.get(rawBytes);
- return DatumFactory.createBytes(rawBytes);
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/CharReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/CharReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/CharReader.java
deleted file mode 100644
index 19dd59c..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/CharReader.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.Datum;
-import tajo.datum.DatumFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class CharReader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- return DatumFactory.createChar(buffer.get());
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/DoubleReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/DoubleReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/DoubleReader.java
deleted file mode 100644
index 021a42d..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/DoubleReader.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.Datum;
-import tajo.datum.DatumFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class DoubleReader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- return DatumFactory.createDouble(buffer.getDouble());
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/FloatReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/FloatReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/FloatReader.java
deleted file mode 100644
index e5f0e03..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/FloatReader.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.Datum;
-import tajo.datum.DatumFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class FloatReader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- return DatumFactory.createFloat(buffer.getFloat());
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IPv4Reader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IPv4Reader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IPv4Reader.java
deleted file mode 100644
index d9094c7..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IPv4Reader.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.Datum;
-import tajo.datum.DatumFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class IPv4Reader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- int size = buffer.getInt();
- byte[] rawBytes = new byte[size];
- buffer.get(rawBytes);
- return DatumFactory.createIPv4(rawBytes);
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/c1c6f83e/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IntReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IntReader.java b/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IntReader.java
deleted file mode 100644
index fa6ad62..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/tajo/storage/hcfile/reader/IntReader.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package tajo.storage.hcfile.reader;
-
-import tajo.datum.Datum;
-import tajo.datum.DatumFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public class IntReader extends TypeReader {
-
- @Override
- public Datum read(ByteBuffer buffer) throws IOException {
- if (buffer.hasRemaining()) {
- return DatumFactory.createInt(buffer.getInt());
- } else {
- return null;
- }
- }
-}