You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2013/10/27 07:40:52 UTC
[1/5] TAJO-287: Refactor TableDesc, TableMeta, and Fragment. (hyunsik)
Updated Branches:
refs/heads/master 5470bcea1 -> 7dc8de283
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/FragmentDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/FragmentDeserializer.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/FragmentDeserializer.java
deleted file mode 100644
index a20d45f..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/FragmentDeserializer.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- *
- */
-package org.apache.tajo.storage.json;
-
-import com.google.gson.*;
-import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.Options;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMetaImpl;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.storage.Fragment;
-
-import java.lang.reflect.Type;
-
-public class FragmentDeserializer implements JsonDeserializer<Fragment> {
-
- @Override
- public Fragment deserialize(JsonElement json, Type type,
- JsonDeserializationContext ctx) throws JsonParseException {
- Gson gson = StorageGsonHelper.getInstance();
- JsonObject fragObj = json.getAsJsonObject();
- JsonObject metaObj = fragObj.get("meta").getAsJsonObject();
- TableMetaImpl meta = new TableMetaImpl(
- gson.fromJson(metaObj.get("schema"), Schema.class),
- gson.fromJson(metaObj.get("storeType"), StoreType.class),
- gson.fromJson(metaObj.get("options"), Options.class));
- Fragment fragment = new Fragment(fragObj.get("tabletId").getAsString(),
- gson.fromJson(fragObj.get("path"), Path.class),
- meta,
- fragObj.get("startOffset").getAsLong(),
- fragObj.get("length").getAsLong());
- return fragment;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/StorageGsonHelper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/StorageGsonHelper.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/StorageGsonHelper.java
deleted file mode 100644
index c95bc73..0000000
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/json/StorageGsonHelper.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- *
- */
-package org.apache.tajo.storage.json;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.json.TableDescAdapter;
-import org.apache.tajo.catalog.json.TableMetaAdapter;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.json.*;
-import org.apache.tajo.util.TUtil;
-
-import java.lang.reflect.Type;
-import java.util.Map;
-
-public class StorageGsonHelper {
- private static Gson gson;
- private static Gson gsonPretty;
-
- private StorageGsonHelper() {
- }
-
- private static Map<Type, GsonSerDerAdapter> registerAdapters() {
- Map<Type, GsonSerDerAdapter> adapters = TUtil.newHashMap();
- adapters.put(Path.class, new PathSerializer());
- adapters.put(TableDesc.class, new TableDescAdapter());
- adapters.put(Class.class, new ClassNameSerializer());
- adapters.put(TableMeta.class, new TableMetaAdapter());
- adapters.put(Datum.class, new DatumAdapter());
- return adapters;
- }
-
- public static Gson getInstance() {
- if (gson == null ) {
- GsonHelper helper = new GsonHelper(registerAdapters());
- gson = helper.getGson();
- }
- return gson;
- }
-
- public static Gson getPrettyInstance() {
- if (gsonPretty == null) {
- GsonBuilder prettyBuilder = new GsonBuilder()
- .setPrettyPrinting()
- .excludeFieldsWithoutExposeAnnotation();
- GsonHelper.registerAdapters(prettyBuilder, registerAdapters());
- gsonPretty = prettyBuilder.create();
- }
-
- return gsonPretty;
- }
-
- public static String toJson(GsonObject object, Class<? extends GsonObject> clazz) {
- return getInstance().toJson(object, clazz);
- }
-
- public static <T extends GsonObject> T fromJson(String json, Class<T> clazz) {
- return getInstance().fromJson(json, clazz);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
index cdc7780..791da37 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFileWrapper.java
@@ -25,11 +25,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.ProtobufDatumFactory;
import org.apache.tajo.storage.*;
+import org.apache.tajo.storage.annotation.ForSplitableStore;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.TUtil;
@@ -47,8 +49,8 @@ public class RCFileWrapper {
private TableStatistics stats = null;
- public RCFileAppender(Configuration conf, TableMeta meta, Path path) throws IOException {
- super(conf, meta, path);
+ public RCFileAppender(Configuration conf, TableMeta meta, Schema schema, Path path) throws IOException {
+ super(conf, meta, schema, path);
}
public void init() throws IOException {
@@ -148,7 +150,7 @@ public class RCFileWrapper {
}
@Override
- public TableStat getStats() {
+ public TableStats getStats() {
if (enabledStats) {
return stats.getTableStat();
} else {
@@ -157,6 +159,7 @@ public class RCFileWrapper {
}
}
+ @ForSplitableStore
public static class RCFileScanner extends FileScanner {
private FileSystem fs;
private RCFile.Reader reader;
@@ -167,9 +170,9 @@ public class RCFileWrapper {
private boolean more;
long end;
- public RCFileScanner(Configuration conf, final TableMeta meta,
- final Fragment fragment) throws IOException {
- super(conf, meta, fragment);
+ public RCFileScanner(Configuration conf, final TableMeta meta, final Schema schema, final Fragment fragment)
+ throws IOException {
+ super(conf, meta, schema, fragment);
fs = fragment.getPath().getFileSystem(conf);
end = fragment.getStartOffset() + fragment.getLength();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
index 22a013d..a6fb6b8 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
@@ -24,8 +24,9 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.storage.FileAppender;
import org.apache.tajo.storage.TableStatistics;
@@ -46,8 +47,8 @@ public class TrevniAppender extends FileAppender {
private TableStatistics stats = null;
private boolean flushed = false;
- public TrevniAppender(Configuration conf, TableMeta meta, Path path) throws IOException {
- super(conf, meta, path);
+ public TrevniAppender(Configuration conf, TableMeta meta, Schema schema, Path path) throws IOException {
+ super(conf, meta, schema, path);
}
public void init() throws IOException {
@@ -60,9 +61,9 @@ public class TrevniAppender extends FileAppender {
fos = fs.create(path);
ColumnMetaData [] trevniMetas =
- new ColumnMetaData[meta.getSchema().getColumnNum()];
+ new ColumnMetaData[schema.getColumnNum()];
int i = 0;
- for (Column column : meta.getSchema().getColumns()) {
+ for (Column column : schema.getColumns()) {
trevniMetas[i++] = new ColumnMetaData(column.getColumnName(),
getType(column.getDataType().getType()));
}
@@ -190,7 +191,7 @@ public class TrevniAppender extends FileAppender {
}
@Override
- public TableStat getStats() {
+ public TableStats getStats() {
if (enabledStats) {
return stats.getTableStat();
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
index 8ffe458..02ce6e7 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
@@ -21,6 +21,7 @@ package org.apache.tajo.storage.trevni;
import com.google.protobuf.Message;
import org.apache.hadoop.conf.Configuration;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.datum.BlobDatum;
import org.apache.tajo.datum.DatumFactory;
@@ -45,8 +46,8 @@ public class TrevniScanner extends FileScanner {
private int [] projectionMap;
private ColumnValues [] columns;
- public TrevniScanner(Configuration conf, TableMeta meta, Fragment fragment) throws IOException {
- super(conf, meta, fragment);
+ public TrevniScanner(Configuration conf, TableMeta meta, Schema schema, Fragment fragment) throws IOException {
+ super(conf, meta, schema, fragment);
reader = new ColumnFileReader(new HadoopInput(fragment.getPath(), conf));
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
index 4529873..8a2d9a5 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.compress.*;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.storage.Fragment;
import org.apache.tajo.storage.LazyTuple;
@@ -67,9 +68,9 @@ public class CSVFileScanner extends FileScannerV2 {
private long totalReadBytesForFetch;
private long totalReadBytesFromDisk;
- public CSVFileScanner(Configuration conf, final TableMeta meta,
- final Fragment fragment) throws IOException {
- super(conf, meta, fragment);
+ public CSVFileScanner(Configuration conf, final TableMeta meta, final Schema schema, final Fragment fragment)
+ throws IOException {
+ super(conf, meta, schema, fragment);
factory = new CompressionCodecFactory(conf);
codec = factory.getCodec(fragment.getPath());
if (isCompress() && !(codec instanceof SplittableCompressionCodec)) {
@@ -81,7 +82,7 @@ public class CSVFileScanner extends FileScannerV2 {
public void init() throws IOException {
// Buffer size, Delimiter
this.bufSize = DEFAULT_BUFFER_SIZE;
- String delim = fragment.getMeta().getOption(DELIMITER, DELIMITER_DEFAULT);
+ String delim = meta.getOption(DELIMITER, DELIMITER_DEFAULT);
this.delimiter = delim.charAt(0);
super.init();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java
index fa757d8..47c4e04 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java
@@ -72,10 +72,11 @@ public abstract class FileScannerV2 implements Scanner {
public FileScannerV2(final Configuration conf,
final TableMeta meta,
+ final Schema schema,
final Fragment fragment) throws IOException {
this.conf = conf;
this.meta = meta;
- this.schema = meta.getSchema();
+ this.schema = schema;
this.fragment = fragment;
this.columnNum = this.schema.getColumnNum();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
index f2da623..a472660 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
@@ -23,11 +23,13 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.storage.Fragment;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.storage.annotation.ForSplitableStore;
import org.apache.tajo.storage.rcfile.BytesRefArrayWritable;
import org.apache.tajo.storage.rcfile.ColumnProjectionUtils;
import org.apache.tajo.util.Bytes;
@@ -36,6 +38,7 @@ import org.apache.tajo.util.TUtil;
import java.io.IOException;
import java.util.ArrayList;
+@ForSplitableStore
public class RCFileScanner extends FileScannerV2 {
private static final Log LOG = LogFactory.getLog(RCFileScanner.class);
@@ -50,10 +53,9 @@ public class RCFileScanner extends FileScannerV2 {
private boolean first = true;
private int maxBytesPerSchedule;
- public RCFileScanner(final Configuration conf,
- final TableMeta meta,
- final Fragment fragment) throws IOException {
- super(conf, meta, fragment);
+ public RCFileScanner(final Configuration conf, final TableMeta meta, final Schema schema, final Fragment fragment)
+ throws IOException {
+ super(conf, meta, schema, fragment);
this.start = fragment.getStartOffset();
this.end = start + fragment.getLength();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
index 0625d28..66eefe6 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
@@ -22,6 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.storage.AbstractStorageManager;
import org.apache.tajo.storage.Fragment;
@@ -53,29 +54,32 @@ public class StorageManagerV2 extends AbstractStorageManager {
}
@Override
- public Scanner getScanner(TableMeta meta, Fragment fragment,
- Schema target) throws IOException {
- Scanner scanner;
-
+ public Class<? extends Scanner> getScannerClass(CatalogProtos.StoreType storeType) throws IOException {
Class<? extends Scanner> scannerClass;
- String handlerName = meta.getStoreType().name().toLowerCase();
+ String handlerName = storeType.name().toLowerCase();
String handlerNameKey = handlerName + "_v2";
scannerClass = SCANNER_HANDLER_CACHE.get(handlerNameKey);
if (scannerClass == null) {
- scannerClass = conf.getClass(
- String.format("tajo.storage.scanner-handler.v2.%s.class",
- meta.getStoreType().name().toLowerCase()), null,
- Scanner.class);
+ scannerClass = conf.getClass(String.format("tajo.storage.scanner-handler.v2.%s.class",
+ storeType.name().toLowerCase()), null, Scanner.class);
SCANNER_HANDLER_CACHE.put(handlerNameKey, scannerClass);
}
+ return scannerClass;
+ }
+
+ @Override
+ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException {
+ Scanner scanner;
+
+ Class<? extends Scanner> scannerClass = getScannerClass(meta.getStoreType());
if (scannerClass == null) {
throw new IOException("Unknown Storage Type: " + meta.getStoreType());
}
- scanner = newScannerInstance(scannerClass, conf, meta, fragment);
+ scanner = newScannerInstance(scannerClass, conf, meta, schema, fragment);
if (scanner.isProjectable()) {
scanner.setTarget(target.toArray());
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
index 401bd9e..f5d3de5 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
@@ -27,7 +27,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.DatumFactory;
@@ -101,11 +101,11 @@ public class TestCompressionStorages {
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
meta.putOption("compression.codec", BZip2Codec.class.getCanonicalName());
Path tablePath = new Path(testDir, "SplitCompression");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
@@ -126,7 +126,7 @@ public class TestCompressionStorages {
}
appender.close();
- TableStat stat = appender.getStats();
+ TableStats stat = appender.getStats();
assertEquals(tupleNum, stat.getNumRows().longValue());
tablePath = tablePath.suffix(extention);
@@ -135,10 +135,10 @@ public class TestCompressionStorages {
long randomNum = (long) (Math.random() * fileLen) + 1;
Fragment[] tablets = new Fragment[2];
- tablets[0] = new Fragment("SplitCompression", tablePath, meta, 0, randomNum);
- tablets[1] = new Fragment("SplitCompression", tablePath, meta, randomNum, (fileLen - randomNum));
+ tablets[0] = new Fragment("SplitCompression", tablePath, 0, randomNum);
+ tablets[1] = new Fragment("SplitCompression", tablePath, randomNum, (fileLen - randomNum));
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[0], schema);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
@@ -147,7 +147,7 @@ public class TestCompressionStorages {
}
scanner.close();
- scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[1], schema);
+ scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema);
scanner.init();
while ((tuple = scanner.next()) != null) {
tupleCnt++;
@@ -162,12 +162,12 @@ public class TestCompressionStorages {
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType);
meta.putOption("compression.codec", codec.getCanonicalName());
String fileName = "Compression_" + codec.getSimpleName();
Path tablePath = new Path(testDir, fileName);
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
@@ -188,15 +188,15 @@ public class TestCompressionStorages {
}
appender.close();
- TableStat stat = appender.getStats();
+ TableStats stat = appender.getStats();
assertEquals(tupleNum, stat.getNumRows().longValue());
tablePath = tablePath.suffix(extension);
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
Fragment[] tablets = new Fragment[1];
- tablets[0] = new Fragment(fileName, tablePath, meta, 0, fileLen);
+ tablets[0] = new Fragment(fileName, tablePath, 0, fileLen);
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[0], schema);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index b869dbb..43c16f6 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@ -26,7 +26,7 @@ import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
@@ -89,10 +89,10 @@ public class TestMergeScanner {
schema.addColumn("age", Type.INT8);
Options options = new Options();
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType, options);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
Path table1Path = new Path(testDir, storeType + "_1.data");
- Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(meta, table1Path);
+ Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, table1Path);
appender1.enableStats();
appender1.init();
int tupleNum = 10000;
@@ -108,13 +108,13 @@ public class TestMergeScanner {
}
appender1.close();
- TableStat stat1 = appender1.getStats();
+ TableStats stat1 = appender1.getStats();
if (stat1 != null) {
assertEquals(tupleNum, stat1.getNumRows().longValue());
}
Path table2Path = new Path(testDir, storeType + "_2.data");
- Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(meta, table2Path);
+ Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, table2Path);
appender2.enableStats();
appender2.init();
@@ -128,7 +128,7 @@ public class TestMergeScanner {
}
appender2.close();
- TableStat stat2 = appender2.getStats();
+ TableStats stat2 = appender2.getStats();
if (stat2 != null) {
assertEquals(tupleNum, stat2.getNumRows().longValue());
}
@@ -137,10 +137,10 @@ public class TestMergeScanner {
FileStatus status1 = fs.getFileStatus(table1Path);
FileStatus status2 = fs.getFileStatus(table2Path);
Fragment[] tablets = new Fragment[2];
- tablets[0] = new Fragment("tablet1", table1Path, meta, 0, status1.getLen());
- tablets[1] = new Fragment("tablet1", table2Path, meta, 0, status2.getLen());
+ tablets[0] = new Fragment("tablet1", table1Path, 0, status1.getLen());
+ tablets[1] = new Fragment("tablet1", table2Path, 0, status2.getLen());
- Scanner scanner = new MergeScanner(conf, meta, TUtil.newList(tablets));
+ Scanner scanner = new MergeScanner(conf, meta, schema, TUtil.newList(tablets));
scanner.init();
int totalCounts = 0;
while (scanner.next() != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
index 5881432..cffb7d9 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
@@ -62,7 +62,7 @@ public class TestStorageManager {
schema.addColumn("age",Type.INT4);
schema.addColumn("name",Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
Tuple[] tuples = new Tuple[4];
for(int i=0; i < tuples.length; i++) {
@@ -75,14 +75,14 @@ public class TestStorageManager {
Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv");
fs.mkdirs(path.getParent());
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, path);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, path);
appender.init();
for(Tuple t : tuples) {
appender.addTuple(t);
}
appender.close();
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, path);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, path);
scanner.init();
int i=0;
while(scanner.next() != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 90c6a6d..d15ceb5 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -28,7 +28,7 @@ import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
@@ -92,9 +92,9 @@ public class TestStorages {
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType);
Path tablePath = new Path(testDir, "Splitable.data");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
int tupleNum = 10000;
@@ -107,7 +107,7 @@ public class TestStorages {
appender.addTuple(vTuple);
}
appender.close();
- TableStat stat = appender.getStats();
+ TableStats stat = appender.getStats();
assertEquals(tupleNum, stat.getNumRows().longValue());
FileStatus status = fs.getFileStatus(tablePath);
@@ -115,10 +115,10 @@ public class TestStorages {
long randomNum = (long) (Math.random() * fileLen) + 1;
Fragment[] tablets = new Fragment[2];
- tablets[0] = new Fragment("Splitable", tablePath, meta, 0, randomNum);
- tablets[1] = new Fragment("Splitable", tablePath, meta, randomNum, (fileLen - randomNum));
+ tablets[0] = new Fragment("Splitable", tablePath, 0, randomNum);
+ tablets[1] = new Fragment("Splitable", tablePath, randomNum, (fileLen - randomNum));
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[0], schema);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
scanner.init();
int tupleCnt = 0;
while (scanner.next() != null) {
@@ -126,7 +126,7 @@ public class TestStorages {
}
scanner.close();
- scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[1], schema);
+ scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema);
scanner.init();
while (scanner.next() != null) {
tupleCnt++;
@@ -144,10 +144,10 @@ public class TestStorages {
schema.addColumn("age", Type.INT8);
schema.addColumn("score", Type.FLOAT4);
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType);
Path tablePath = new Path(testDir, "testProjection.data");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
int tupleNum = 10000;
VTuple vTuple;
@@ -162,12 +162,12 @@ public class TestStorages {
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
- Fragment fragment = new Fragment("testReadAndWrite", tablePath, meta, 0, status.getLen());
+ Fragment fragment = new Fragment("testReadAndWrite", tablePath, 0, status.getLen());
Schema target = new Schema();
target.addColumn("age", Type.INT8);
target.addColumn("score", Type.FLOAT4);
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, fragment, target);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, target);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
@@ -202,10 +202,10 @@ public class TestStorages {
schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
Options options = new Options();
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType, options);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
QueryId queryid = new QueryId("12345", 5);
@@ -232,8 +232,8 @@ public class TestStorages {
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
- Fragment fragment = new Fragment("table", tablePath, meta, 0, status.getLen());
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, fragment);
+ Fragment fragment = new Fragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
index 233fc29..ec89aa4 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
@@ -70,10 +70,10 @@ public class TestBSTIndex {
@Test
public void testFindValueInCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = new Path(testDir, "FindValueInCSV.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple;
for(int i = 0 ; i < TUPLE_NUM; i ++ ) {
@@ -89,7 +89,7 @@ public class TestBSTIndex {
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
- Fragment tablet = new Fragment("table1_1", status.getPath(), meta, 0, fileLen);
+ Fragment tablet = new Fragment("table1_1", status.getPath(), 0, fileLen);
SortSpec [] sortKeys = new SortSpec[2];
sortKeys[0] = new SortSpec(schema.getColumnByFQN("long"), true, false);
@@ -108,7 +108,7 @@ public class TestBSTIndex {
creater.setLoadNum(LOAD_NUM);
creater.open();
- SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
Tuple keyTuple;
@@ -131,7 +131,7 @@ public class TestBSTIndex {
tuple = new VTuple(keySchema.getColumnNum());
BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindValueInCSV.idx"), keySchema, comp);
reader.open();
- scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
for(int i = 0 ; i < TUPLE_NUM -1 ; i ++) {
@@ -156,10 +156,11 @@ public class TestBSTIndex {
@Test
public void testBuildIndexWithAppender() throws IOException {
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = new Path(testDir, "BuildIndexWithAppender.csv");
- FileAppender appender = (FileAppender) StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ FileAppender appender = (FileAppender) StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema,
+ tablePath);
appender.init();
SortSpec [] sortKeys = new SortSpec[2];
@@ -201,13 +202,13 @@ public class TestBSTIndex {
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
- Fragment tablet = new Fragment("table1_1", status.getPath(), meta, 0, fileLen);
+ Fragment tablet = new Fragment("table1_1", status.getPath(), 0, fileLen);
tuple = new VTuple(keySchema.getColumnNum());
BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "BuildIndexWithAppender.idx"),
keySchema, comp);
reader.open();
- SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
for(int i = 0 ; i < TUPLE_NUM -1 ; i ++) {
@@ -232,10 +233,10 @@ public class TestBSTIndex {
@Test
public void testFindOmittedValueInCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = StorageUtil.concatPath(testDir, "FindOmittedValueInCSV.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple;
for(int i = 0 ; i < TUPLE_NUM; i += 2 ) {
@@ -250,7 +251,7 @@ public class TestBSTIndex {
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
- Fragment tablet = new Fragment("table1_1", status.getPath(), meta, 0, status.getLen());
+ Fragment tablet = new Fragment("table1_1", status.getPath(), 0, status.getLen());
SortSpec [] sortKeys = new SortSpec[2];
sortKeys[0] = new SortSpec(schema.getColumnByFQN("long"), true, false);
@@ -268,7 +269,7 @@ public class TestBSTIndex {
creater.setLoadNum(LOAD_NUM);
creater.open();
- SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
Tuple keyTuple;
@@ -300,10 +301,10 @@ public class TestBSTIndex {
@Test
public void testFindNextKeyValueInCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = new Path(testDir, "FindNextKeyValueInCSV.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple;
for(int i = 0 ; i < TUPLE_NUM; i ++ ) {
@@ -319,7 +320,7 @@ public class TestBSTIndex {
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
- Fragment tablet = new Fragment("table1_1", status.getPath(), meta, 0, fileLen);
+ Fragment tablet = new Fragment("table1_1", status.getPath(), 0, fileLen);
SortSpec [] sortKeys = new SortSpec[2];
sortKeys[0] = new SortSpec(schema.getColumnByFQN("int"), true, false);
@@ -337,7 +338,7 @@ public class TestBSTIndex {
creater.setLoadNum(LOAD_NUM);
creater.open();
- SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
Tuple keyTuple;
@@ -360,7 +361,7 @@ public class TestBSTIndex {
BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindNextKeyValueInCSV.idx"),
keySchema, comp);
reader.open();
- scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
Tuple result;
@@ -388,10 +389,10 @@ public class TestBSTIndex {
@Test
public void testFindNextKeyOmittedValueInCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = new Path(testDir, "FindNextKeyOmittedValueInCSV.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple;
for(int i = 0 ; i < TUPLE_NUM; i+=2) {
@@ -407,7 +408,7 @@ public class TestBSTIndex {
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
- Fragment tablet = new Fragment("table1_1", status.getPath(), meta, 0, fileLen);
+ Fragment tablet = new Fragment("table1_1", status.getPath(), 0, fileLen);
SortSpec [] sortKeys = new SortSpec[2];
sortKeys[0] = new SortSpec(schema.getColumnByFQN("int"), true, false);
@@ -425,7 +426,7 @@ public class TestBSTIndex {
creater.setLoadNum(LOAD_NUM);
creater.open();
- SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
Tuple keyTuple;
@@ -448,7 +449,7 @@ public class TestBSTIndex {
BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindNextKeyOmittedValueInCSV.idx"),
keySchema, comp);
reader.open();
- scanner = StorageManagerFactory.getSeekableScanner(conf, meta, tablet, meta.getSchema());
+ scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema);
scanner.init();
Tuple result;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
index 00a72e4..96d78ea 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
@@ -72,12 +72,12 @@ public class TestSingleCSVFileBSTIndex {
@Test
public void testFindValueInSingleCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv");
fs.mkdirs(tablePath.getParent());
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple;
for (int i = 0; i < TUPLE_NUM; i++) {
@@ -93,7 +93,7 @@ public class TestSingleCSVFileBSTIndex {
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
- Fragment tablet = new Fragment("table1_1", status.getPath(), meta, 0, fileLen);
+ Fragment tablet = new Fragment("table1_1", status.getPath(), 0, fileLen);
SortSpec[] sortKeys = new SortSpec[2];
sortKeys[0] = new SortSpec(schema.getColumnByFQN("long"), true, false);
@@ -111,7 +111,7 @@ public class TestSingleCSVFileBSTIndex {
creater.setLoadNum(LOAD_NUM);
creater.open();
- SeekableScanner fileScanner = new CSVScanner(conf, meta, tablet);
+ SeekableScanner fileScanner = new CSVScanner(conf, meta, schema, tablet);
fileScanner.init();
Tuple keyTuple;
long offset;
@@ -135,7 +135,7 @@ public class TestSingleCSVFileBSTIndex {
BSTIndexReader reader = bst.getIndexReader(new Path(testDir,
"FindValueInCSV.idx"), keySchema, comp);
reader.open();
- fileScanner = new CSVScanner(conf, meta, tablet);
+ fileScanner = new CSVScanner(conf, meta, schema, tablet);
fileScanner.init();
for (int i = 0; i < TUPLE_NUM - 1; i++) {
tuple.put(0, DatumFactory.createInt8(i));
@@ -161,12 +161,12 @@ public class TestSingleCSVFileBSTIndex {
@Test
public void testFindNextKeyValueInSingleCSV() throws IOException {
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV",
"table1.csv");
fs.mkdirs(tablePath.getParent());
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple;
for(int i = 0 ; i < TUPLE_NUM; i ++ ) {
@@ -182,7 +182,7 @@ public class TestSingleCSVFileBSTIndex {
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
- Fragment tablet = new Fragment("table1_1", status.getPath(), meta, 0, fileLen);
+ Fragment tablet = new Fragment("table1_1", status.getPath(), 0, fileLen);
SortSpec [] sortKeys = new SortSpec[2];
sortKeys[0] = new SortSpec(schema.getColumnByFQN("int"), true, false);
@@ -200,7 +200,7 @@ public class TestSingleCSVFileBSTIndex {
creater.setLoadNum(LOAD_NUM);
creater.open();
- SeekableScanner fileScanner = new CSVScanner(conf, meta, tablet);
+ SeekableScanner fileScanner = new CSVScanner(conf, meta, schema, tablet);
fileScanner.init();
Tuple keyTuple;
long offset;
@@ -221,7 +221,7 @@ public class TestSingleCSVFileBSTIndex {
BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindNextKeyValueInCSV.idx"), keySchema, comp);
reader.open();
- fileScanner = new CSVScanner(conf, meta, tablet);
+ fileScanner = new CSVScanner(conf, meta, schema, tablet);
fileScanner.init();
Tuple result;
for(int i = 0 ; i < TUPLE_NUM -1 ; i ++) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
index e5dc00f..a3e0360 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java
@@ -27,7 +27,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.DatumFactory;
@@ -104,11 +104,11 @@ public class TestCSVCompression {
schema.addColumn("id", TajoDataTypes.Type.INT4);
schema.addColumn("age", TajoDataTypes.Type.INT8);
- TableMeta meta = CatalogUtil.newTableMeta(schema, CatalogProtos.StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
meta.putOption("compression.codec", BZip2Codec.class.getCanonicalName());
Path tablePath = new Path(testDir, "SplitCompression");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
@@ -129,7 +129,7 @@ public class TestCSVCompression {
}
appender.close();
- TableStat stat = appender.getStats();
+ TableStats stat = appender.getStats();
assertEquals(tupleNum, stat.getNumRows().longValue());
tablePath = tablePath.suffix(extention);
@@ -138,10 +138,10 @@ public class TestCSVCompression {
long randomNum = (long) (Math.random() * fileLen) + 1;
Fragment[] tablets = new Fragment[2];
- tablets[0] = new Fragment("SplitCompression", tablePath, meta, 0, randomNum);
- tablets[1] = new Fragment("SplitCompression", tablePath, meta, randomNum, (fileLen - randomNum));
+ tablets[0] = new Fragment("SplitCompression", tablePath, 0, randomNum);
+ tablets[1] = new Fragment("SplitCompression", tablePath, randomNum, (fileLen - randomNum));
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[0], schema);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
@@ -150,7 +150,7 @@ public class TestCSVCompression {
}
scanner.close();
- scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[1], schema);
+ scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema);
scanner.init();
while ((tuple = scanner.next()) != null) {
tupleCnt++;
@@ -160,17 +160,18 @@ public class TestCSVCompression {
assertEquals(tupleNum, tupleCnt);
}
- private void storageCompressionTest(CatalogProtos.StoreType storeType, Class<? extends CompressionCodec> codec) throws IOException {
+ private void storageCompressionTest(CatalogProtos.StoreType storeType, Class<? extends CompressionCodec> codec)
+ throws IOException {
Schema schema = new Schema();
schema.addColumn("id", TajoDataTypes.Type.INT4);
schema.addColumn("age", TajoDataTypes.Type.INT8);
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType);
meta.putOption("compression.codec", codec.getCanonicalName());
String fileName = "Compression_" + codec.getSimpleName();
Path tablePath = new Path(testDir, fileName);
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
@@ -191,15 +192,15 @@ public class TestCSVCompression {
}
appender.close();
- TableStat stat = appender.getStats();
+ TableStats stat = appender.getStats();
assertEquals(tupleNum, stat.getNumRows().longValue());
tablePath = tablePath.suffix(extension);
FileStatus status = fs.getFileStatus(tablePath);
long fileLen = status.getLen();
Fragment[] tablets = new Fragment[1];
- tablets[0] = new Fragment(fileName, tablePath, meta, 0, fileLen);
+ tablets[0] = new Fragment(fileName, tablePath, 0, fileLen);
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[0], schema);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
scanner.init();
int tupleCnt = 0;
while (scanner.next() != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
index f647a4c..99d28bd 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java
@@ -66,7 +66,7 @@ public class TestCSVScanner {
schema.addColumn("age", TajoDataTypes.Type.INT4);
schema.addColumn("name", TajoDataTypes.Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(schema, CatalogProtos.StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
Tuple[] tuples = new Tuple[4];
for(int i=0; i < tuples.length; i++) {
@@ -79,14 +79,14 @@ public class TestCSVScanner {
Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv");
fs.mkdirs(path.getParent());
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, path);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, path);
appender.init();
for(Tuple t : tuples) {
appender.addTuple(t);
}
appender.close();
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, path);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, path);
scanner.init();
int i=0;
Tuple tuple = null;
@@ -103,12 +103,12 @@ public class TestCSVScanner {
schema.addColumn("age", TajoDataTypes.Type.INT4);
schema.addColumn("name", TajoDataTypes.Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(schema, CatalogProtos.StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
Path path = StorageUtil.concatPath(testDir, "testPartitionFile", "table.csv");
fs.mkdirs(path.getParent());
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, path);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, path);
appender.init();
String keyValue = "";
@@ -144,8 +144,8 @@ public class TestCSVScanner {
long startOffset = (64 * 1024 * 1024) * scanCount;
long length = Math.min(64 * 1024 * 1024, fileLength - startOffset);
- Fragment fragment = new Fragment("Test", path, meta, startOffset, length, null, null);
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, fragment, schema);
+ Fragment fragment = new Fragment("Test", path, startOffset, length, null, null);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, schema);
scanner.init();
Tuple tuple = null;
while( (tuple = scanner.next()) != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
index 91dd935..89dec07 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
@@ -26,7 +26,7 @@ import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
@@ -92,9 +92,9 @@ public class TestStorages {
schema.addColumn("id", Type.INT4);
schema.addColumn("age", Type.INT8);
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType);
Path tablePath = new Path(testDir, "Splitable.data");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
appender.init();
int tupleNum = 10000;
@@ -107,7 +107,7 @@ public class TestStorages {
appender.addTuple(vTuple);
}
appender.close();
- TableStat stat = appender.getStats();
+ TableStats stat = appender.getStats();
assertEquals(tupleNum, stat.getNumRows().longValue());
FileStatus status = fs.getFileStatus(tablePath);
@@ -115,10 +115,10 @@ public class TestStorages {
long randomNum = (long) (Math.random() * fileLen) + 1;
Fragment[] tablets = new Fragment[2];
- tablets[0] = new Fragment("Splitable", tablePath, meta, 0, randomNum);
- tablets[1] = new Fragment("Splitable", tablePath, meta, randomNum, (fileLen - randomNum));
+ tablets[0] = new Fragment("Splitable", tablePath, 0, randomNum);
+ tablets[1] = new Fragment("Splitable", tablePath, randomNum, (fileLen - randomNum));
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[0], schema);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
scanner.init();
int tupleCnt = 0;
while (scanner.next() != null) {
@@ -126,7 +126,7 @@ public class TestStorages {
}
scanner.close();
- scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, tablets[1], schema);
+ scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema);
scanner.init();
while (scanner.next() != null) {
tupleCnt++;
@@ -144,10 +144,10 @@ public class TestStorages {
schema.addColumn("age", Type.INT8);
schema.addColumn("score", Type.FLOAT4);
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType);
Path tablePath = new Path(testDir, "testProjection.data");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
int tupleNum = 10000;
VTuple vTuple;
@@ -162,12 +162,12 @@ public class TestStorages {
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
- Fragment fragment = new Fragment("testReadAndWrite", tablePath, meta, 0, status.getLen());
+ Fragment fragment = new Fragment("testReadAndWrite", tablePath, 0, status.getLen());
Schema target = new Schema();
target.addColumn("age", Type.INT8);
target.addColumn("score", Type.FLOAT4);
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, fragment, target);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, target);
scanner.init();
int tupleCnt = 0;
Tuple tuple;
@@ -201,10 +201,10 @@ public class TestStorages {
schema.addColumn("col12", Type.NULL);
Options options = new Options();
- TableMeta meta = CatalogUtil.newTableMeta(schema, storeType, options);
+ TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.init();
Tuple tuple = new VTuple(12);
@@ -227,8 +227,8 @@ public class TestStorages {
appender.close();
FileStatus status = fs.getFileStatus(tablePath);
- Fragment fragment = new Fragment("table", tablePath, meta, 0, status.getLen());
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, fragment);
+ Fragment fragment = new Fragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
Tuple retrieved;
[4/5] TAJO-287: Refactor TableDesc, TableMeta, and Fragment. (hyunsik)
Posted by hy...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
index 88bc74f..904b4bc 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java
@@ -21,8 +21,6 @@ package org.apache.tajo.catalog;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.catalog.statistics.ColumnStat;
-import org.apache.tajo.catalog.statistics.TableStat;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.junit.Before;
import org.junit.Test;
@@ -31,31 +29,10 @@ import static org.junit.Assert.*;
public class TestTableMeta {
TableMeta meta = null;
- Schema schema = null;
@Before
- public void setUp() {
- schema = new Schema();
- schema.addColumn("name", Type.BLOB);
- schema.addColumn("addr", Type.TEXT);
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
-
- TableStat stat = new TableStat();
- stat.setNumRows(957685);
- stat.setNumBytes(1023234);
- stat.setNumBlocks(3123);
- stat.setNumPartitions(5);
- stat.setAvgRows(80000);
-
- int numCols = 2;
- ColumnStat[] cols = new ColumnStat[numCols];
- for (int i = 0; i < numCols; i++) {
- cols[i] = new ColumnStat(schema.getColumn(i));
- cols[i].setNumDistVals(1024 * i);
- cols[i].setNumNulls(100 * i);
- stat.addColumnStat(cols[i]);
- }
- meta.setStat(stat);
+ public void setUp() {
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
}
@Test
@@ -63,9 +40,9 @@ public class TestTableMeta {
Schema schema1 = new Schema();
schema1.addColumn("name", Type.BLOB);
schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
+ TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV);
- TableMeta meta2 = new TableMetaImpl(meta1.getProto());
+ TableMeta meta2 = new TableMeta(meta1.getProto());
assertEquals(meta1, meta2);
}
@@ -74,10 +51,9 @@ public class TestTableMeta {
Schema schema1 = new Schema();
schema1.addColumn("name", Type.BLOB);
schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
+ TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV);
- TableMetaImpl meta2 = (TableMetaImpl) meta1.clone();
- assertEquals(meta1.getSchema(), meta2.getSchema());
+ TableMeta meta2 = (TableMeta) meta1.clone();
assertEquals(meta1.getStoreType(), meta2.getStoreType());
assertEquals(meta1, meta2);
}
@@ -87,7 +63,7 @@ public class TestTableMeta {
Schema schema1 = new Schema();
schema1.addColumn("name", Type.BLOB);
schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
+ TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV);
TableMeta meta2 = (TableMeta) meta1.clone();
@@ -100,49 +76,11 @@ public class TestTableMeta {
}
@Test
- public void testGetSchema() {
- Schema schema2 = new Schema();
- schema2.addColumn("name", Type.BLOB);
- schema2.addColumn("addr", Type.TEXT);
-
- assertEquals(schema, schema2);
- }
-
- @Test
- public void testSetSchema() {
- Schema schema2 = new Schema();
- schema2.addColumn("name", Type.BLOB);
- schema2.addColumn("addr", Type.TEXT);
- schema2.addColumn("age", Type.INT4);
-
- assertNotSame(meta.getSchema(), schema2);
- meta.setSchema(schema2);
- assertEquals(meta.getSchema(), schema2);
- }
-
- @Test
public void testEqualsObject() {
Schema schema2 = new Schema();
schema2.addColumn("name", Type.BLOB);
schema2.addColumn("addr", Type.TEXT);
- TableMeta meta2 = CatalogUtil.newTableMeta(schema2, StoreType.CSV);
-
- TableStat stat = new TableStat();
- stat.setNumRows(957685);
- stat.setNumBytes(1023234);
- stat.setNumBlocks(3123);
- stat.setNumPartitions(5);
- stat.setAvgRows(80000);
-
- int numCols = 2;
- ColumnStat[] cols = new ColumnStat[numCols];
- for (int i = 0; i < numCols; i++) {
- cols[i] = new ColumnStat(schema2.getColumn(i));
- cols[i].setNumDistVals(1024 * i);
- cols[i].setNumNulls(100 * i);
- stat.addColumnStat(cols[i]);
- }
- meta2.setStat(stat);
+ TableMeta meta2 = CatalogUtil.newTableMeta(StoreType.CSV);
assertTrue(meta.equals(meta2));
@@ -152,7 +90,7 @@ public class TestTableMeta {
@Test
public void testGetProto() {
TableProto proto = meta.getProto();
- TableMeta newMeta = new TableMetaImpl(proto);
+ TableMeta newMeta = new TableMeta(proto);
assertEquals(meta, newMeta);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java
index c9dea3c..6c1c311 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java
@@ -31,52 +31,52 @@ public class TestColumnStat {
@Test
public final void testColumnStat() {
- ColumnStat stat = new ColumnStat(new Column("test", Type.INT8));
+ ColumnStats stat = new ColumnStats(new Column("test", Type.INT8));
stat.setNumDistVals(1000);
stat.setNumNulls(999);
assertTrue(1000 == stat.getNumDistValues());
assertTrue(999 == stat.getNumNulls());
- ColumnStat stat2 = new ColumnStat(stat.getProto());
+ ColumnStats stat2 = new ColumnStats(stat.getProto());
assertTrue(1000 == stat2.getNumDistValues());
assertTrue(999 == stat2.getNumNulls());
}
@Test
public final void testEqualsObject() {
- ColumnStat stat = new ColumnStat(new Column("test", Type.INT8));
+ ColumnStats stat = new ColumnStats(new Column("test", Type.INT8));
stat.setNumDistVals(1000);
stat.setNumNulls(999);
stat.setMinValue(DatumFactory.createInt8(5));
stat.setMaxValue(DatumFactory.createInt8(10));
- ColumnStat stat2 = new ColumnStat(stat.getProto());
+ ColumnStats stat2 = new ColumnStats(stat.getProto());
assertEquals(stat, stat2);
}
@Test
public final void testJson() throws CloneNotSupportedException {
- ColumnStat stat = new ColumnStat(new Column("test", Type.INT8));
+ ColumnStats stat = new ColumnStats(new Column("test", Type.INT8));
stat.setNumDistVals(1000);
stat.setNumNulls(999);
stat.setMinValue(DatumFactory.createInt8(5));
stat.setMaxValue(DatumFactory.createInt8(10));
String json = stat.toJson();
- ColumnStat fromJson = CatalogGsonHelper.fromJson(json, ColumnStat.class);
+ ColumnStats fromJson = CatalogGsonHelper.fromJson(json, ColumnStats.class);
assertEquals(stat, fromJson);
}
@Test
public final void testClone() throws CloneNotSupportedException {
- ColumnStat stat = new ColumnStat(new Column("test", Type.INT8));
+ ColumnStats stat = new ColumnStats(new Column("test", Type.INT8));
stat.setNumDistVals(1000);
stat.setNumNulls(999);
stat.setMinValue(DatumFactory.createInt8(5));
stat.setMaxValue(DatumFactory.createInt8(10));
- ColumnStat stat2 = (ColumnStat) stat.clone();
+ ColumnStats stat2 = (ColumnStats) stat.clone();
assertEquals(stat, stat2);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatisticsUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatisticsUtil.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatisticsUtil.java
index 8019fec..5820e0d 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatisticsUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatisticsUtil.java
@@ -58,9 +58,9 @@ public class TestStatisticsUtil {
@Test
public void testEmptyAggregate() {
- TableStat stat1 = new TableStat();
- TableStat stat2 = new TableStat();
- TableStat stat3 = new TableStat();
+ TableStats stat1 = new TableStats();
+ TableStats stat2 = new TableStats();
+ TableStats stat3 = new TableStats();
assertNotNull(StatisticsUtil.aggregateTableStat(
Lists.newArrayList(stat1, stat2, stat3)));
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java
index 5372291..3e8da9a 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java
@@ -29,7 +29,7 @@ import static org.junit.Assert.assertTrue;
public class TestTableStat {
@Test
public final void testTableStat() throws CloneNotSupportedException {
- TableStat stat = new TableStat();
+ TableStats stat = new TableStats();
stat.setNumRows(957685);
stat.setNumBytes(1023234);
stat.setNumBlocks(3123);
@@ -37,9 +37,9 @@ public class TestTableStat {
stat.setAvgRows(80000);
int numCols = 3;
- ColumnStat[] cols = new ColumnStat[numCols];
+ ColumnStats[] cols = new ColumnStats[numCols];
for (int i = 0; i < numCols; i++) {
- cols[i] = new ColumnStat(new Column("col_" + i, Type.INT8));
+ cols[i] = new ColumnStats(new Column("col_" + i, Type.INT8));
cols[i].setNumDistVals(1024 * i);
cols[i].setNumNulls(100 * i);
stat.addColumnStat(cols[i]);
@@ -55,18 +55,18 @@ public class TestTableStat {
assertEquals(cols[i], stat.getColumnStats().get(i));
}
- TableStat stat2 = new TableStat(stat.getProto());
+ TableStats stat2 = new TableStats(stat.getProto());
tableStatEquals(stat, stat2);
- TableStat stat3 = (TableStat) stat.clone();
+ TableStats stat3 = (TableStats) stat.clone();
tableStatEquals(stat, stat3);
String json = stat.toJson();
- TableStat fromJson = CatalogGsonHelper.fromJson(json, TableStat.class);
+ TableStats fromJson = CatalogGsonHelper.fromJson(json, TableStats.class);
tableStatEquals(stat, fromJson);
}
- public void tableStatEquals(TableStat s1, TableStat s2) {
+ public void tableStatEquals(TableStats s1, TableStats s2) {
assertEquals(s1.getNumRows(), s2.getNumRows());
assertEquals(s1.getNumBlocks(), s2.getNumBlocks());
assertEquals(s1.getNumPartitions(), s2.getNumPartitions());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 672912a..a6ef576 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -18,7 +18,6 @@
package org.apache.tajo.catalog;
-import com.google.common.base.Preconditions;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
@@ -183,10 +182,10 @@ public class CatalogServer extends AbstractService {
throw new NoSuchTableException(tableId);
}
TableDesc desc = store.getTable(tableId);
- SchemaProto schemaProto = desc.getMeta().getSchema().getProto();
+ SchemaProto schemaProto = desc.getSchema().getProto();
SchemaProto qualifiedSchema = CatalogUtil.getQualfiedSchema(tableId, schemaProto);
- desc.getMeta().setSchema(new Schema(qualifiedSchema));
- return (TableDescProto) desc.getProto();
+ desc.setSchema(new Schema(qualifiedSchema));
+ return desc.getProto();
} catch (IOException ioe) {
// TODO - handle exception
LOG.error(ioe);
@@ -229,8 +228,6 @@ public class CatalogServer extends AbstractService {
@Override
public BoolProto addTable(RpcController controller, TableDescProto tableDesc)
throws ServiceException {
- Preconditions.checkArgument(tableDesc.hasId(), "Must be set to the table name");
- Preconditions.checkArgument(tableDesc.hasPath(), "Must be set to the table URI");
wlock.lock();
try {
@@ -239,12 +236,12 @@ public class CatalogServer extends AbstractService {
}
// rewrite schema
- TableProto.Builder metaBuilder = TableProto.newBuilder(tableDesc.getMeta());
- metaBuilder.setSchema(tableDesc.getMeta().getSchema());
TableDescProto.Builder descBuilder = TableDescProto.newBuilder(tableDesc);
- descBuilder.setMeta(metaBuilder.build());
+ descBuilder.setMeta(tableDesc.getMeta());
+ descBuilder.setSchema(tableDesc.getSchema());
- store.addTable(new TableDescImpl(descBuilder.build()));
+
+ store.addTable(new TableDesc(descBuilder.build()));
} catch (IOException ioe) {
LOG.error(ioe.getMessage(), ioe);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index 33e3bff..9e0762f 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -30,7 +30,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.InternalException;
@@ -208,7 +208,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
String colSql;
int columnId = 0;
- for (Column col : table.getMeta().getSchema().getColumns()) {
+ for (Column col : table.getSchema().getColumns()) {
colSql = columnToSQL(tid, table, columnId, col);
if (LOG.isDebugEnabled()) {
LOG.debug(colSql);
@@ -235,11 +235,11 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
CatalogUtil.closeSQLWrapper(pstmt);
}
- if (table.getMeta().getStat() != null) {
+ if (table.getStats() != null) {
sql = "INSERT INTO " + TB_STATISTICS + " (" + C_TABLE_ID + ", num_rows, num_bytes) "
+ "VALUES ('" + table.getName() + "', "
- + table.getMeta().getStat().getNumRows() + ","
- + table.getMeta().getStat().getNumBytes() + ")";
+ + table.getStats().getNumRows() + ","
+ + table.getStats().getNumBytes() + ")";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
@@ -361,7 +361,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
Path path = null;
StoreType storeType = null;
Options options;
- TableStat stat = null;
+ TableStats stat = null;
try {
String sql =
@@ -445,7 +445,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
res = stmt.executeQuery(sql);
if (res.next()) {
- stat = new TableStat();
+ stat = new TableStats();
stat.setNumRows(res.getLong("num_rows"));
stat.setNumBytes(res.getLong("num_bytes"));
}
@@ -455,11 +455,11 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
CatalogUtil.closeSQLWrapper(res, stmt);
}
- TableMeta meta = new TableMetaImpl(schema, storeType, options);
+ TableMeta meta = new TableMeta(storeType, options);
+ TableDesc table = new TableDesc(tableName, schema, meta, path);
if (stat != null) {
- meta.setStat(stat);
+ table.setStats(stat);
}
- TableDesc table = new TableDescImpl(tableName, meta, path);
return table;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
index a1377ab..e1fea50 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java
@@ -28,7 +28,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.InternalException;
@@ -295,7 +295,7 @@ public class DerbyStore extends AbstractDBStore {
String colSql;
int columnId = 0;
- for (Column col : table.getMeta().getSchema().getColumns()) {
+ for (Column col : table.getSchema().getColumns()) {
colSql = columnToSQL(tid, table, columnId, col);
if (LOG.isDebugEnabled()) {
LOG.debug(colSql);
@@ -314,11 +314,11 @@ public class DerbyStore extends AbstractDBStore {
stmt.addBatch(optSql);
}
stmt.executeBatch();
- if (table.getMeta().getStat() != null) {
+ if (table.getStats() != null) {
sql = "INSERT INTO " + TB_STATISTICS + " (" + C_TABLE_ID + ", num_rows, num_bytes) "
+ "VALUES ('" + table.getName() + "', "
- + table.getMeta().getStat().getNumRows() + ","
- + table.getMeta().getStat().getNumBytes() + ")";
+ + table.getStats().getNumRows() + ","
+ + table.getStats().getNumBytes() + ")";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
@@ -453,7 +453,7 @@ public class DerbyStore extends AbstractDBStore {
Path path = null;
StoreType storeType = null;
Options options;
- TableStat stat = null;
+ TableStats stat = null;
try {
rlock.lock();
@@ -540,7 +540,7 @@ public class DerbyStore extends AbstractDBStore {
res = stmt.executeQuery(sql);
if (res.next()) {
- stat = new TableStat();
+ stat = new TableStats();
stat.setNumRows(res.getLong("num_rows"));
stat.setNumBytes(res.getLong("num_bytes"));
}
@@ -550,11 +550,11 @@ public class DerbyStore extends AbstractDBStore {
CatalogUtil.closeSQLWrapper(res);
}
- TableMeta meta = new TableMetaImpl(schema, storeType, options);
+ TableMeta meta = new TableMeta(storeType, options);
+ TableDesc table = new TableDesc(tableName, schema, meta, path);
if (stat != null) {
- meta.setStat(stat);
+ table.setStats(stat);
}
- TableDesc table = new TableDescImpl(tableName, meta, path);
return table;
} catch (SQLException se) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
index 7323393..77b258e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
@@ -33,14 +33,10 @@ import java.util.List;
import java.util.Map;
public class MemStore implements CatalogStore {
- private final Map<String,TableDesc> tables
- = Maps.newHashMap();
- private final Map<String, FunctionDesc> functions
- = Maps.newHashMap();
- private final Map<String, IndexDescProto> indexes
- = Maps.newHashMap();
- private final Map<String, IndexDescProto> indexesByColumn
- = Maps.newHashMap();
+ private final Map<String,TableDesc> tables = Maps.newHashMap();
+ private final Map<String, FunctionDesc> functions = Maps.newHashMap();
+ private final Map<String, IndexDescProto> indexes = Maps.newHashMap();
+ private final Map<String, IndexDescProto> indexesByColumn = Maps.newHashMap();
public MemStore(Configuration conf) {
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index b692597..b644e69 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -92,8 +92,8 @@ public class TestCatalog {
schema1.addColumn(FieldName2, Type.INT4);
schema1.addColumn(FieldName3, Type.INT8);
- TableMeta info = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
- TableDesc desc = new TableDescImpl();
+ TableMeta info = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc desc = new TableDesc();
desc.setMeta(info);
catalog.addTable(desc);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestDBStore.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestDBStore.java
index 71672be..12e175c 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestDBStore.java
@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.catalog.store.AbstractDBStore;
import org.apache.tajo.catalog.store.DerbyStore;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -69,15 +69,15 @@ public class TestDBStore {
String tableName = "addedtable";
Options opts = new Options();
opts.put("file.delimiter", ",");
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV, opts);
- TableDesc desc = new TableDescImpl(tableName, meta, new Path(CommonTestingUtil.getTestDir(), "addedtable"));
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
+ TableDesc desc = new TableDesc(tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), "addedtable"));
assertFalse(store.existTable(tableName));
store.addTable(desc);
assertTrue(store.existTable(tableName));
TableDesc retrieved = store.getTable(tableName);
// Schema order check
- assertSchemaOrder(desc.getMeta().getSchema(), retrieved.getMeta().getSchema());
+ assertSchemaOrder(desc.getSchema(), retrieved.getSchema());
store.deleteTable(tableName);
assertFalse(store.existTable(tableName));
}
@@ -93,23 +93,23 @@ public class TestDBStore {
String tableName = "gettable";
Options opts = new Options();
opts.put("file.delimiter", ",");
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV, opts);
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts);
- TableStat stat = new TableStat();
+ TableStats stat = new TableStats();
stat.setNumRows(957685);
stat.setNumBytes(1023234);
- meta.setStat(stat);
- TableDesc desc = new TableDescImpl(tableName, meta, new Path(CommonTestingUtil.getTestDir(), "gettable"));
+ TableDesc desc = new TableDesc(tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), "gettable"));
+ desc.setStats(stat);
store.addTable(desc);
TableDesc retrieved = store.getTable(tableName);
assertEquals(",", retrieved.getMeta().getOption("file.delimiter"));
assertEquals(desc, retrieved);
- assertTrue(957685 == desc.getMeta().getStat().getNumRows());
- assertTrue(1023234 == desc.getMeta().getStat().getNumBytes());
+ assertTrue(957685 == desc.getStats().getNumRows());
+ assertTrue(1023234 == desc.getStats().getNumBytes());
// Schema order check
- assertSchemaOrder(desc.getMeta().getSchema(), retrieved.getMeta().getSchema());
+ assertSchemaOrder(desc.getSchema(), retrieved.getSchema());
store.deleteTable(tableName);
}
@@ -124,8 +124,8 @@ public class TestDBStore {
int numTables = 5;
for (int i = 0; i < numTables; i++) {
String tableName = "tableA_" + i;
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- TableDesc desc = new TableDescImpl(tableName, meta,
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc desc = new TableDesc(tableName, schema, meta,
new Path(CommonTestingUtil.getTestDir(), "tableA_" + i));
store.addTable(desc);
}
@@ -205,8 +205,8 @@ public class TestDBStore {
String tableName = "indexed";
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- return new TableDescImpl(tableName, meta, new Path(CommonTestingUtil.getTestDir(), "indexed"));
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ return new TableDesc(tableName, schema, meta, new Path(CommonTestingUtil.getTestDir(), "indexed"));
}
public static void assertSchemaOrder(Schema s1, Schema s2) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
index 7b33b09..291b83f 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
@@ -165,10 +165,10 @@ public class TPCH extends BenchmarkSet {
}
private void loadTable(String tableName) throws ServiceException {
- TableMeta meta = CatalogUtil.newTableMeta(getSchema(tableName), StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
meta.putOption(CSVFile.DELIMITER, "|");
try {
- tajo.createExternalTable(tableName, new Path(dataDir, tableName), meta);
+ tajo.createExternalTable(tableName, getSchema(tableName), new Path(dataDir, tableName), meta);
} catch (SQLException s) {
throw new ServiceException(s);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
index 984f6e4..9aea8a5 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -29,7 +29,7 @@ import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.QueryStatus;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
@@ -367,7 +367,7 @@ public class TajoCli {
ResultSetMetaData rsmd = res.getMetaData();
TableDesc desc = client.getResultDesc(queryId);
- TableStat stat = desc.getMeta().getStat();
+ TableStats stat = desc.getStats();
String volume = FileUtil.humanReadableByteCount(stat.getNumBytes(), false);
long resultRows = stat.getNumRows();
sout.println("result: " + desc.getPath() + ", " + resultRows + " rows (" + volume + ")");
@@ -438,10 +438,10 @@ public class TajoCli {
sb.append("\ntable name: ").append(desc.getName()).append("\n");
sb.append("table path: ").append(desc.getPath()).append("\n");
sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n");
- if (desc.getMeta().getStat() != null) {
- sb.append("number of rows: ").append(desc.getMeta().getStat().getNumRows()).append("\n");
+ if (desc.getStats() != null) {
+ sb.append("number of rows: ").append(desc.getStats().getNumRows()).append("\n");
sb.append("volume: ").append(
- FileUtil.humanReadableByteCount(desc.getMeta().getStat().getNumBytes(),
+ FileUtil.humanReadableByteCount(desc.getStats().getNumBytes(),
true)).append("\n");
}
sb.append("Options: \n");
@@ -452,8 +452,8 @@ public class TajoCli {
sb.append("\n");
sb.append("schema: \n");
- for(int i = 0; i < desc.getMeta().getSchema().getColumnNum(); i++) {
- Column col = desc.getMeta().getSchema().getColumn(i);
+ for(int i = 0; i < desc.getSchema().getColumnNum(); i++) {
+ Column col = desc.getSchema().getColumn(i);
sb.append(col.getColumnName()).append("\t").append(col.getDataType().getType());
if (col.getDataType().hasLength()) {
sb.append("(").append(col.getDataType().getLength()).append(")");
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetImpl.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetImpl.java
index b19bb9d..b713f71 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetImpl.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetImpl.java
@@ -68,12 +68,12 @@ public class ResultSetImpl implements ResultSet {
this.tajoClient = tajoClient;
this.queryId = queryId;
this.desc = desc;
- this.schema = desc.getMeta().getSchema();
+ this.schema = desc.getSchema();
if(desc != null) {
fs = desc.getPath().getFileSystem(conf);
- this.totalRow = desc.getMeta().getStat() != null ? desc.getMeta().getStat().getNumRows() : 0;
+ this.totalRow = desc.getStats() != null ? desc.getStats().getNumRows() : 0;
Collection<Fragment> frags = getFragments(desc.getMeta(), desc.getPath());
- scanner = new MergeScanner(conf, desc.getMeta(), frags);
+ scanner = new MergeScanner(conf, desc.getMeta(), schema, frags);
}
init();
}
@@ -107,7 +107,7 @@ public class ResultSetImpl implements ResultSet {
if (files[i].getLen() == 0) {
continue;
}
- fraglist.add(new Fragment(tbname + "_" + i, files[i].getPath(), meta, 0l, files[i].getLen()));
+ fraglist.add(new Fragment(tbname + "_" + i, files[i].getPath(), 0l, files[i].getLen()));
}
return fraglist;
}
@@ -675,7 +675,7 @@ public class ResultSetImpl implements ResultSet {
*/
@Override
public ResultSetMetaData getMetaData() throws SQLException {
- return new ResultSetMetaDataImpl(desc.getMeta());
+ return new ResultSetMetaDataImpl(desc);
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetMetaDataImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetMetaDataImpl.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetMetaDataImpl.java
index d8c8ad1..6e2d6e9 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetMetaDataImpl.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/ResultSetMetaDataImpl.java
@@ -21,7 +21,7 @@
*/
package org.apache.tajo.client;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.exception.UnsupportedException;
@@ -31,10 +31,11 @@ import java.sql.SQLException;
import java.sql.Types;
public class ResultSetMetaDataImpl implements ResultSetMetaData {
- private TableMeta meta;
+ private TableDesc desc;
+
- public ResultSetMetaDataImpl(TableMeta meta) {
- this.meta = meta;
+ public ResultSetMetaDataImpl(TableDesc desc) {
+ this.desc = desc;
}
/* (non-Javadoc)
@@ -67,7 +68,7 @@ public class ResultSetMetaDataImpl implements ResultSetMetaData {
*/
@Override
public String getColumnClassName(int column) throws SQLException {
- return meta.getSchema().getColumn(column - 1).getClass().getName();
+ return desc.getSchema().getColumn(column - 1).getClass().getName();
}
/* (non-Javadoc)
@@ -75,7 +76,7 @@ public class ResultSetMetaDataImpl implements ResultSetMetaData {
*/
@Override
public int getColumnCount() throws SQLException {
- return meta.getSchema().getColumnNum();
+ return desc.getSchema().getColumnNum();
}
/* (non-Javadoc)
@@ -91,7 +92,7 @@ public class ResultSetMetaDataImpl implements ResultSetMetaData {
*/
@Override
public String getColumnLabel(int column) throws SQLException {
- return meta.getSchema().getColumn(column - 1).getQualifiedName();
+ return desc.getSchema().getColumn(column - 1).getQualifiedName();
}
/* (non-Javadoc)
@@ -99,7 +100,7 @@ public class ResultSetMetaDataImpl implements ResultSetMetaData {
*/
@Override
public String getColumnName(int column) throws SQLException {
- return meta.getSchema().getColumn(column - 1).getColumnName();
+ return desc.getSchema().getColumn(column - 1).getColumnName();
}
/* (non-Javadoc)
@@ -108,7 +109,7 @@ public class ResultSetMetaDataImpl implements ResultSetMetaData {
@Override
public int getColumnType(int column) throws SQLException {
// TODO
- DataType type = meta.getSchema().getColumn(column - 1).getDataType();
+ DataType type = desc.getSchema().getColumn(column - 1).getDataType();
switch (type.getType()) {
case BOOLEAN:
return Types.BOOLEAN;
@@ -146,7 +147,7 @@ public class ResultSetMetaDataImpl implements ResultSetMetaData {
*/
@Override
public String getColumnTypeName(int column) throws SQLException {
- return meta.getSchema().getColumn(column - 1).
+ return desc.getSchema().getColumn(column - 1).
getDataType().getClass().getCanonicalName();
}
@@ -179,7 +180,7 @@ public class ResultSetMetaDataImpl implements ResultSetMetaData {
*/
@Override
public String getTableName(int column) throws SQLException {
- return meta.getSchema().getColumn(column - 1).getQualifier();
+ return desc.getSchema().getColumn(column - 1).getQualifier();
}
/* (non-Javadoc)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/TajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/TajoClient.java
index d94c99f..f73fe2e 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/TajoClient.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/client/TajoClient.java
@@ -27,6 +27,7 @@ import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.annotation.ThreadSafe;
import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.conf.TajoConf;
@@ -280,32 +281,19 @@ public class TajoClient {
return tajoMasterService.existTable(null, builder.build()).getValue();
}
- public TableDesc attachTable(String name, String path)
- throws ServiceException {
- AttachTableRequest.Builder builder = AttachTableRequest.newBuilder();
- builder.setName(name);
- builder.setPath(path);
- TableResponse res = tajoMasterService.attachTable(null, builder.build());
- return CatalogUtil.newTableDesc(res.getTableDesc());
- }
-
- public TableDesc attachTable(String name, Path path)
- throws ServiceException {
- return attachTable(name, path.toString());
- }
-
public boolean detachTable(String name) throws ServiceException {
StringProto.Builder builder = StringProto.newBuilder();
builder.setValue(name);
return tajoMasterService.detachTable(null, builder.build()).getValue();
}
- public TableDesc createExternalTable(String name, Path path, TableMeta meta)
+ public TableDesc createExternalTable(String name, Schema schema, Path path, TableMeta meta)
throws SQLException, ServiceException {
CreateTableRequest.Builder builder = CreateTableRequest.newBuilder();
builder.setName(name);
- builder.setPath(path.toUri().toString());
+ builder.setSchema(schema.getProto());
builder.setMeta(meta.getProto());
+ builder.setPath(path.toUri().toString());
TableResponse res = tajoMasterService.createExternalTable(null, builder.build());
if (res.getResultCode() == ResultCode.OK) {
return CatalogUtil.newTableDesc(res.getTableDesc());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
index 784c539..4dfb314 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java
@@ -21,18 +21,15 @@ package org.apache.tajo.engine.json;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.json.GsonObject;
-import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.engine.function.AggFunction;
import org.apache.tajo.catalog.function.Function;
-import org.apache.tajo.engine.function.GeneralFunction;
import org.apache.tajo.catalog.json.FunctionAdapter;
-import org.apache.tajo.catalog.json.TableDescAdapter;
import org.apache.tajo.catalog.json.TableMetaAdapter;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.engine.eval.EvalNode;
+import org.apache.tajo.engine.function.AggFunction;
+import org.apache.tajo.engine.function.GeneralFunction;
import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.json.*;
import org.apache.tajo.util.TUtil;
@@ -50,7 +47,6 @@ public class CoreGsonHelper {
private static Map<Type, GsonSerDerAdapter> registerAdapters() {
Map<Type, GsonSerDerAdapter> adapters = TUtil.newHashMap();
adapters.put(Path.class, new PathSerializer());
- adapters.put(TableDesc.class, new TableDescAdapter());
adapters.put(Class.class, new ClassNameSerializer());
adapters.put(LogicalNode.class, new LogicalNodeAdapter());
adapters.put(EvalNode.class, new EvalNodeAdapter());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 84cbe14..79f02d1 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -37,13 +37,13 @@ import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.eval.*;
+import org.apache.tajo.engine.exception.InvalidQueryException;
+import org.apache.tajo.engine.exception.UndefinedFunctionException;
import org.apache.tajo.engine.exception.VerifyException;
import org.apache.tajo.engine.function.AggFunction;
import org.apache.tajo.engine.function.GeneralFunction;
import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock;
import org.apache.tajo.engine.planner.logical.*;
-import org.apache.tajo.engine.exception.InvalidQueryException;
-import org.apache.tajo.engine.exception.UndefinedFunctionException;
import org.apache.tajo.engine.utils.SchemaUtil;
import org.apache.tajo.exception.InternalException;
@@ -184,11 +184,11 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
try {
FileSystem fs = desc.getPath().getFileSystem(new Configuration());
FileStatus status = fs.getFileStatus(desc.getPath());
- if (desc.getMeta().getStat() != null && (status.isDirectory() || status.isFile())) {
+ if (desc.getStats() != null && (status.isDirectory() || status.isFile())) {
ContentSummary summary = fs.getContentSummary(desc.getPath());
if (summary != null) {
long volume = summary.getLength();
- desc.getMeta().getStat().setNumBytes(volume);
+ desc.getStats().setNumBytes(volume);
}
}
} catch (Throwable t) {
@@ -848,7 +848,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
targetSchema.addColumn(targetColumn);
}
} else {
- Schema targetTableSchema = desc.getMeta().getSchema();
+ Schema targetTableSchema = desc.getSchema();
for (int i = 0; i < subQuery.getOutSchema().getColumnNum(); i++) {
targetSchema.addColumn(targetTableSchema.getColumn(i));
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 37f0f9e..f7540e7 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -131,8 +131,8 @@ public class GlobalPlanner {
public static ScanNode buildInputExecutor(LogicalPlan plan, DataChannel channel) {
Preconditions.checkArgument(channel.getSchema() != null,
"Channel schema (" + channel.getSrcId().getId() +" -> "+ channel.getTargetId().getId()+") is not initialized");
- TableMeta meta = new TableMetaImpl(channel.getSchema(), channel.getStoreType(), new Options());
- TableDesc desc = new TableDescImpl(channel.getSrcId().toString(), meta, new Path("/"));
+ TableMeta meta = new TableMeta(channel.getStoreType(), new Options());
+ TableDesc desc = new TableDesc(channel.getSrcId().toString(), channel.getSchema(), meta, new Path("/"));
return new ScanNode(plan.newPID(), desc);
}
@@ -169,14 +169,14 @@ public class GlobalPlanner {
ScanNode leftScan = (ScanNode) leftNode;
ScanNode rightScan = (ScanNode) rightNode;
- TableMeta leftMeta = leftScan.getTableDesc().getMeta();
- TableMeta rightMeta = rightScan.getTableDesc().getMeta();
+ TableDesc leftDesc = leftScan.getTableDesc();
+ TableDesc rightDesc = rightScan.getTableDesc();
long broadcastThreshold = conf.getLongVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD);
- if (leftMeta.getStat().getNumBytes() < broadcastThreshold) {
+ if (leftDesc.getStats().getNumBytes() < broadcastThreshold) {
leftBroadcasted = true;
}
- if (rightMeta.getStat().getNumBytes() < broadcastThreshold) {
+ if (rightDesc.getStats().getNumBytes() < broadcastThreshold) {
rightBroadcasted = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java
index 05d7e19..d80393b 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java
@@ -231,8 +231,8 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
public static double getCost(LogicalNode node) {
if (node instanceof ScanNode) {
ScanNode scanNode = (ScanNode) node;
- if (scanNode.getTableDesc().getMeta().getStat() != null) {
- return ((ScanNode)node).getTableDesc().getMeta().getStat().getNumBytes();
+ if (scanNode.getTableDesc().getStats() != null) {
+ return ((ScanNode)node).getTableDesc().getStats().getNumBytes();
} else {
return Long.MAX_VALUE;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
index 4430609..2d81180 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
@@ -61,7 +61,7 @@ public class BSTIndexScanExec extends PhysicalExec {
this.datum = datum;
this.fileScanner = StorageManagerFactory.getSeekableScanner(context.getConf(),
- fragment.getMeta(), fragment, outSchema);
+ scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema);
this.fileScanner.init();
this.projector = new Projector(inSchema, outSchema, scanNode.getTargets());
this.evalContexts = projector.renew();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
index 12bd30b..99df92e 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
@@ -20,13 +20,13 @@ package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.worker.TaskAttemptContext;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.logical.SortNode;
import org.apache.tajo.storage.*;
+import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
import java.util.*;
@@ -55,7 +55,7 @@ public class ExternalSortExec extends SortExec {
this.sortTmpDir = new Path(context.getWorkDir(), UUID.randomUUID().toString());
this.localFS = FileSystem.getLocal(context.getConf());
- meta = CatalogUtil.newTableMeta(inSchema, StoreType.ROWFILE);
+ meta = CatalogUtil.newTableMeta(StoreType.ROWFILE);
}
public void init() throws IOException {
@@ -69,13 +69,13 @@ public class ExternalSortExec extends SortExec {
private void sortAndStoreChunk(int chunkId, List<Tuple> tupleSlots)
throws IOException {
- TableMeta meta = CatalogUtil.newTableMeta(inSchema, StoreType.RAW);
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.RAW);
Collections.sort(tupleSlots, getComparator());
// TODO - RawFile requires the local file path.
// So, I add the scheme 'file:/' to path. But, it should be improved.
Path localPath = new Path(sortTmpDir + "/0_" + chunkId);
- appender = new RawFile.RawFileAppender(context.getConf(), meta, localPath);
+ appender = new RawFile.RawFileAppender(context.getConf(), meta, inSchema, localPath);
appender.init();
for (Tuple t : tupleSlots) {
@@ -147,7 +147,7 @@ public class ExternalSortExec extends SortExec {
Path leftChunk = getChunkPath(level, chunkId);
Path rightChunk = getChunkPath(level, chunkId + 1);
- appender = new RawFile.RawFileAppender(context.getConf(), meta, nextChunk);
+ appender = new RawFile.RawFileAppender(context.getConf(), meta, inSchema, nextChunk);
appender.init();
merge(appender, leftChunk, rightChunk);
@@ -167,7 +167,7 @@ public class ExternalSortExec extends SortExec {
}
Path result = getChunkPath(level, 0);
- this.result = new RawFile.RawFileScanner(context.getConf(), meta, result);
+ this.result = new RawFile.RawFileScanner(context.getConf(), meta, plan.getInSchema(), result);
sorted = true;
}
@@ -176,10 +176,10 @@ public class ExternalSortExec extends SortExec {
private void merge(RawFile.RawFileAppender appender, Path left, Path right)
throws IOException {
- RawFile.RawFileScanner leftScan = new RawFile.RawFileScanner(context.getConf(), meta, left);
+ RawFile.RawFileScanner leftScan = new RawFile.RawFileScanner(context.getConf(), meta, plan.getInSchema(), left);
RawFile.RawFileScanner rightScan =
- new RawFile.RawFileScanner(context.getConf(), meta, right);
+ new RawFile.RawFileScanner(context.getConf(), meta, plan.getInSchema(), right);
Tuple leftTuple = leftScan.next();
Tuple rightTuple = rightScan.next();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/IndexedStoreExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/IndexedStoreExec.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/IndexedStoreExec.java
index c9aced4..596c470 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/IndexedStoreExec.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/IndexedStoreExec.java
@@ -68,11 +68,11 @@ public class IndexedStoreExec extends UnaryPhysicalExec {
Path storeTablePath = new Path(context.getWorkDir(), "output");
LOG.info("Output data directory: " + storeTablePath);
this.meta = CatalogUtil
- .newTableMeta(this.outSchema, CatalogProtos.StoreType.CSV);
+ .newTableMeta(CatalogProtos.StoreType.CSV);
FileSystem fs = new RawLocalFileSystem();
fs.mkdirs(storeTablePath);
this.appender = (FileAppender) StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta,
- new Path(storeTablePath, "output"));
+ outSchema, new Path(storeTablePath, "output"));
this.appender.enableStats();
this.appender.init();
this.indexWriter = bst.getIndexWriter(new Path(storeTablePath, "index"),
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/PartitionedStoreExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/PartitionedStoreExec.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/PartitionedStoreExec.java
index 0ca6265..2e53229 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/PartitionedStoreExec.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/PartitionedStoreExec.java
@@ -25,15 +25,15 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.tajo.worker.TaskAttemptContext;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.StatisticsUtil;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.engine.planner.logical.StoreTableNode;
import org.apache.tajo.storage.*;
+import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
import java.text.NumberFormat;
@@ -66,7 +66,7 @@ public final class PartitionedStoreExec extends UnaryPhysicalExec {
super(context, plan.getInSchema(), plan.getOutSchema(), child);
Preconditions.checkArgument(plan.hasPartitionKey());
this.plan = plan;
- this.meta = CatalogUtil.newTableMeta(this.outSchema, StoreType.CSV);
+ this.meta = CatalogUtil.newTableMeta(StoreType.CSV);
// about the partitions
this.numPartitions = this.plan.getNumPartitions();
@@ -98,7 +98,7 @@ public final class PartitionedStoreExec extends UnaryPhysicalExec {
FileStatus status = fs.getFileStatus(dataFile);
LOG.info("File size: " + status.getLen());
}
- appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, dataFile);
+ appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, dataFile);
appender.enableStats();
appender.init();
appenderMap.put(partition, appender);
@@ -124,7 +124,7 @@ public final class PartitionedStoreExec extends UnaryPhysicalExec {
appender.addTuple(tuple);
}
- List<TableStat> statSet = new ArrayList<TableStat>();
+ List<TableStats> statSet = new ArrayList<TableStats>();
for (Map.Entry<Integer, Appender> entry : appenderMap.entrySet()) {
int partNum = entry.getKey();
Appender app = entry.getValue();
@@ -137,7 +137,7 @@ public final class PartitionedStoreExec extends UnaryPhysicalExec {
}
// Collect and aggregated statistics data
- TableStat aggregated = StatisticsUtil.aggregateTableStat(statSet);
+ TableStats aggregated = StatisticsUtil.aggregateTableStat(statSet);
context.setResultStats(aggregated);
return null;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
index 6840bb2..2045602 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
@@ -47,7 +47,7 @@ public class SeqScanExec extends PhysicalExec {
private EvalContext [] evalContexts;
public SeqScanExec(TaskAttemptContext context, AbstractStorageManager sm,
- ScanNode plan, Fragment[] fragments) throws IOException {
+ ScanNode plan, Fragment [] fragments) throws IOException {
super(context, plan.getInSchema(), plan.getOutSchema());
this.plan = plan;
@@ -88,11 +88,11 @@ public class SeqScanExec extends PhysicalExec {
this.evalContexts = projector.renew();
if (fragments.length > 1) {
- this.scanner = new MergeScanner(context.getConf(), fragments[0].getMeta(),
+ this.scanner = new MergeScanner(context.getConf(), plan.getTableDesc().getMeta(), plan.getTableSchema(),
TUtil.newList(fragments));
} else {
this.scanner = StorageManagerFactory.getStorageManager(
- context.getConf()).getScanner(fragments[0].getMeta(), fragments[0], projected);
+ context.getConf()).getScanner(plan.getTableDesc().getMeta(), plan.getTableSchema(), fragments[0], projected);
}
scanner.init();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
index 68ca8fc..673e0b5 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
@@ -24,11 +24,14 @@ package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.tajo.worker.TaskAttemptContext;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.engine.planner.logical.StoreTableNode;
-import org.apache.tajo.storage.*;
+import org.apache.tajo.storage.Appender;
+import org.apache.tajo.storage.StorageManagerFactory;
+import org.apache.tajo.storage.StorageUtil;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
@@ -54,19 +57,20 @@ public class StoreTableExec extends UnaryPhysicalExec {
TableMeta meta;
if (plan.hasOptions()) {
- meta = CatalogUtil.newTableMeta(outSchema, plan.getStorageType(), plan.getOptions());
+ meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions());
} else {
- meta = CatalogUtil.newTableMeta(outSchema, plan.getStorageType());
+ meta = CatalogUtil.newTableMeta(plan.getStorageType());
}
if (context.isInterQuery()) {
Path storeTablePath = new Path(context.getWorkDir(), "out");
FileSystem fs = new RawLocalFileSystem();
fs.mkdirs(storeTablePath);
- appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta,
+ appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema,
StorageUtil.concatPath(storeTablePath, "0"));
} else {
- appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, context.getOutputPath());
+ appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema,
+ context.getOutputPath());
}
appender.enableStats();
appender.init();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
index 53a8449..ed529a9 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
@@ -26,7 +26,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.catalog.statistics.ColumnStat;
+import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.planner.PlannerUtil;
@@ -364,9 +364,9 @@ public class TupleUtil {
.toTuple(schema, endBytes));
}
- public static TupleRange columnStatToRange(Schema schema, Schema target, List<ColumnStat> colStats) {
- Map<Column, ColumnStat> statSet = Maps.newHashMap();
- for (ColumnStat stat : colStats) {
+ public static TupleRange columnStatToRange(Schema schema, Schema target, List<ColumnStats> colStats) {
+ Map<Column, ColumnStats> statSet = Maps.newHashMap();
+ for (ColumnStats stat : colStats) {
statSet.put(stat.getColumn(), stat);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
index a92b1a0..505fd71 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -33,7 +33,7 @@ import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.exception.AlreadyExistsTableException;
import org.apache.tajo.catalog.exception.NoSuchTableException;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.eval.ConstEval;
@@ -240,11 +240,9 @@ public class GlobalEngine extends AbstractService {
TableMeta meta;
if (createTable.hasOptions()) {
- meta = CatalogUtil.newTableMeta(createTable.getSchema(),
- createTable.getStorageType(), createTable.getOptions());
+ meta = CatalogUtil.newTableMeta(createTable.getStorageType(), createTable.getOptions());
} else {
- meta = CatalogUtil.newTableMeta(createTable.getSchema(),
- createTable.getStorageType());
+ meta = CatalogUtil.newTableMeta(createTable.getStorageType());
}
if(!createTable.isExternal()){
@@ -254,10 +252,11 @@ public class GlobalEngine extends AbstractService {
Preconditions.checkState(createTable.hasPath(), "ERROR: LOCATION must be given.");
}
- return createTableOnDirectory(createTable.getTableName(), meta, createTable.getPath(), true);
+ return createTableOnDirectory(createTable.getTableName(), createTable.getSchema(), meta,
+ createTable.getPath(), true);
}
- public TableDesc createTableOnDirectory(String tableName, TableMeta meta, Path path, boolean isCreated)
+ public TableDesc createTableOnDirectory(String tableName, Schema schema, TableMeta meta, Path path, boolean isCreated)
throws IOException {
if (catalog.existsTable(tableName)) {
throw new AlreadyExistsTableException(tableName);
@@ -281,14 +280,13 @@ public class GlobalEngine extends AbstractService {
LOG.error("Cannot calculate the size of the relation", e);
}
- TableStat stat = new TableStat();
- stat.setNumBytes(totalSize);
- meta.setStat(stat);
-
- TableDesc desc = CatalogUtil.newTableDesc(tableName, meta, path);
+ TableStats stats = new TableStats();
+ stats.setNumBytes(totalSize);
+ TableDesc desc = CatalogUtil.newTableDesc(tableName, schema, meta, path);
+ desc.setStats(stats);
catalog.addTable(desc);
- LOG.info("Table " + desc.getName() + " is created (" + desc.getMeta().getStat().getNumBytes() + ")");
+ LOG.info("Table " + desc.getName() + " is created (" + desc.getStats().getNumBytes() + ")");
return desc;
}
@@ -435,7 +433,7 @@ public class GlobalEngine extends AbstractService {
if (insertNode.hasTargetTable()) { // if a target table is given, it computes the proper schema.
storeNode.getOptions().putAll(insertNode.getTargetTable().getMeta().toMap());
- Schema targetTableSchema = insertNode.getTargetTable().getMeta().getSchema();
+ Schema targetTableSchema = insertNode.getTargetTable().getSchema();
Schema targetProjectedSchema = insertNode.getTargetSchema();
int [] targetColumnIds = new int[targetProjectedSchema.getColumnNum()];
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 6b7d8d3..d1faf4f 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -31,10 +31,7 @@ import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.exception.AlreadyExistsTableException;
import org.apache.tajo.catalog.exception.NoSuchTableException;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
-import org.apache.tajo.catalog.statistics.TableStat;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.ipc.ClientProtos;
@@ -46,7 +43,6 @@ import org.apache.tajo.master.querymaster.QueryInProgress;
import org.apache.tajo.master.querymaster.QueryInfo;
import org.apache.tajo.master.querymaster.QueryJobManager;
import org.apache.tajo.rpc.BlockingRpcServer;
-import org.apache.tajo.rpc.RemoteException;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto;
import org.apache.tajo.util.NetUtils;
@@ -288,12 +284,12 @@ public class TajoMasterClientService extends AbstractService {
if (catalog.existsTable(name)) {
return TableResponse.newBuilder()
.setResultCode(ResultCode.OK)
- .setTableDesc((TableDescProto) catalog.getTableDesc(name).getProto())
+ .setTableDesc(catalog.getTableDesc(name).getProto())
.build();
} else {
return TableResponse.newBuilder()
.setResultCode(ResultCode.ERROR)
- .setErrorMessage("No such a table: " + request.getTableName())
+ .setErrorMessage("ERROR: no such a table: " + request.getTableName())
.build();
}
}
@@ -309,27 +305,13 @@ public class TajoMasterClientService extends AbstractService {
throw new IOException("No such a directory: " + path);
}
- TableMeta meta = new TableMetaImpl(request.getMeta());
-
- if (meta.getStat() == null) {
- meta.setStat(new TableStat());
- }
-
- TableStat stat = meta.getStat();
- long totalSize;
- try {
- totalSize = fs.getContentSummary(path).getSpaceConsumed();
- } catch (IOException e) {
- String message =
- "Cannot get the volume of the table \"" + request.getName() + "\" from " + request.getPath();
- LOG.warn(message);
- throw new IOException(message, e);
- }
- stat.setNumBytes(totalSize);
+ Schema schema = new Schema(request.getSchema());
+ TableMeta meta = new TableMeta(request.getMeta());
TableDesc desc;
try {
- desc = context.getGlobalEngine().createTableOnDirectory(request.getName(), meta, path, false);
+ desc = context.getGlobalEngine().createTableOnDirectory(request.getName(), schema, meta, path,
+ false);
} catch (Exception e) {
return TableResponse.newBuilder()
.setResultCode(ResultCode.ERROR)
@@ -338,7 +320,7 @@ public class TajoMasterClientService extends AbstractService {
return TableResponse.newBuilder()
.setResultCode(ResultCode.OK)
- .setTableDesc((TableDescProto) desc.getProto()).build();
+ .setTableDesc(desc.getProto()).build();
} catch (IOException ioe) {
return TableResponse.newBuilder()
.setResultCode(ResultCode.ERROR)
@@ -355,53 +337,6 @@ public class TajoMasterClientService extends AbstractService {
}
@Override
- public TableResponse attachTable(RpcController controller,
- AttachTableRequest request)
- throws ServiceException {
-
- TableDesc desc;
- if (catalog.existsTable(request.getName())) {
- throw new AlreadyExistsTableException(request.getName());
- }
-
- Path tablePath = new Path(request.getPath());
-
- LOG.info(tablePath.toUri());
-
- TableMeta meta;
- try {
- meta = TableUtil.getTableMeta(conf, tablePath);
- } catch (IOException e) {
- throw new RemoteException(e);
- }
-
- if (meta.getStat() == null) {
- long totalSize;
- try {
- FileSystem fs = tablePath.getFileSystem(conf);
- totalSize = fs.getContentSummary(tablePath).getSpaceConsumed();
- } catch (IOException e) {
- LOG.error("Cannot get the volume of the table", e);
- return null;
- }
-
- meta = new TableMetaImpl(meta.getProto());
- TableStat stat = new TableStat();
- stat.setNumBytes(totalSize);
- meta.setStat(stat);
- }
-
- desc = new TableDescImpl(request.getName(), meta, tablePath);
- catalog.addTable(desc);
- LOG.info("Table " + desc.getName() + " is attached ("
- + meta.getStat().getNumBytes() + ")");
-
- return TableResponse.newBuilder().
- setTableDesc((TableDescProto) desc.getProto())
- .build();
- }
-
- @Override
public BoolProto detachTable(RpcController controller,
StringProto tableNameProto)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/SubQuerySucceeEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/SubQuerySucceeEvent.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/SubQuerySucceeEvent.java
index e02196a..5262a07 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/SubQuerySucceeEvent.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/event/SubQuerySucceeEvent.java
@@ -19,18 +19,10 @@
package org.apache.tajo.master.event;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.master.querymaster.SubQueryState;
public class SubQuerySucceeEvent extends SubQueryCompletedEvent {
- private final TableMeta tableMeta;
-
- public SubQuerySucceeEvent(final ExecutionBlockId id, TableMeta tableMeta) {
+ public SubQuerySucceeEvent(final ExecutionBlockId id) {
super(id, SubQueryState.SUCCEEDED);
- this.tableMeta = tableMeta;
- }
-
- public TableMeta getTableMeta() {
- return tableMeta;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
index ce945b4..9b3c998 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -28,16 +28,15 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.*;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoProtos.QueryState;
import org.apache.tajo.catalog.CatalogService;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableDescImpl;
-import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.planner.global.ExecutionBlock;
@@ -317,7 +316,8 @@ public class Query implements EventHandler<QueryEvent> {
if (query.checkQueryForCompleted() == QueryState.QUERY_SUCCEEDED) {
DataChannel finalChannel = masterPlan.getChannel(castEvent.getExecutionBlockId(), nextBlock.getId());
Path finalOutputDir = commitOutputData(query);
- TableDesc finalTableDesc = buildOrUpdateResultTableDesc(query, castEvent.getExecutionBlockId(), finalOutputDir);
+ TableDesc finalTableDesc = buildOrUpdateResultTableDesc(query, castEvent.getExecutionBlockId(),
+ finalOutputDir);
QueryContext queryContext = query.context.getQueryContext();
CatalogService catalog = query.context.getQueryMasterContext().getWorkerContext().getCatalog();
@@ -380,15 +380,16 @@ public class Query implements EventHandler<QueryEvent> {
}
TableMeta meta = subQuery.getTableMeta();
+ TableStats stats = subQuery.getTableStat();
try {
FileSystem fs = finalOutputDir.getFileSystem(query.systemConf);
ContentSummary directorySummary = fs.getContentSummary(finalOutputDir);
- if(meta.getStat() == null) meta.setStat(new TableStat());
- meta.getStat().setNumBytes(directorySummary.getLength());
+ stats.setNumBytes(directorySummary.getLength());
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
- TableDesc outputTableDesc = new TableDescImpl(outputTableName, meta, finalOutputDir);
+ TableDesc outputTableDesc = new TableDesc(outputTableName, subQuery.getSchema(), meta, finalOutputDir);
+ outputTableDesc.setStats(stats);
TableDesc finalTableDesc = outputTableDesc;
// If a query has a target table, a TableDesc is updated.
@@ -397,7 +398,7 @@ public class Query implements EventHandler<QueryEvent> {
CatalogService catalog = query.context.getQueryMasterContext().getWorkerContext().getCatalog();
Preconditions.checkNotNull(catalog, "CatalogService is NULL");
TableDesc updatingTable = catalog.getTableDesc(outputTableDesc.getName());
- updatingTable.getMeta().setStat(meta.getStat());
+ updatingTable.setStats(stats);
finalTableDesc = updatingTable;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
index 5a30c04..ff0d120 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
@@ -18,7 +18,6 @@
package org.apache.tajo.master.querymaster;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
@@ -29,7 +28,7 @@ import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.QueryUnitId;
import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.ipc.TajoWorkerProtocol.Partition;
import org.apache.tajo.master.TaskState;
@@ -59,7 +58,7 @@ public class QueryUnit implements EventHandler<TaskEvent> {
private Map<String, Set<URI>> fetchMap;
private List<Partition> partitions;
- private TableStat stats;
+ private TableStats stats;
private List<DataLocation> dataLocations;
private final boolean isLeafTask;
private List<IntermediateEntry> intermediateData;
@@ -283,7 +282,7 @@ public class QueryUnit implements EventHandler<TaskEvent> {
return str;
}
- public void setStats(TableStat stats) {
+ public void setStats(TableStats stats) {
this.stats = stats;
}
@@ -291,7 +290,7 @@ public class QueryUnit implements EventHandler<TaskEvent> {
this.partitions = Collections.unmodifiableList(partitions);
}
- public TableStat getStats() {
+ public TableStats getStats() {
return this.stats;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
index f01cb75..2304784 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.state.*;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.TajoProtos.TaskAttemptState;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.ipc.TajoWorkerProtocol.Partition;
import org.apache.tajo.ipc.TajoWorkerProtocol.TaskCompletionReport;
import org.apache.tajo.master.event.*;
@@ -192,7 +192,7 @@ public class QueryUnitAttempt implements EventHandler<TaskAttemptEvent> {
this.getQueryUnit().setIntermediateData(partitions);
}
if (report.hasResultStats()) {
- this.getQueryUnit().setStats(new TableStat(report.getResultStats()));
+ this.getQueryUnit().setStats(new TableStats(report.getResultStats()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index 178e9b5..d0cca20 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -23,22 +23,21 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.StatisticsUtil;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.PlannerUtil;
import org.apache.tajo.engine.planner.RangePartitionAlgorithm;
import org.apache.tajo.engine.planner.UniformRangePartition;
+import org.apache.tajo.engine.planner.global.DataChannel;
+import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.utils.TupleUtil;
import org.apache.tajo.exception.InternalException;
-import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry;
import org.apache.tajo.storage.AbstractStorageManager;
import org.apache.tajo.storage.Fragment;
@@ -77,7 +76,7 @@ public class Repartitioner {
Path tablePath;
Fragment [] fragments = new Fragment[2];
- TableStat [] stats = new TableStat[2];
+ TableStats[] stats = new TableStats[2];
// initialize variables from the child operators
for (int i =0; i < 2; i++) {
@@ -90,13 +89,12 @@ public class Repartitioner {
tablePath = storageManager.getTablePath(scans[i].getTableName());
stats[i] = masterContext.getSubQuery(childBlocks[i].getId()).getTableStat();
- fragments[i] = new Fragment(scans[i].getCanonicalName(), tablePath,
- CatalogUtil.newTableMeta(scans[i].getInSchema(), StoreType.CSV), 0, 0);
+ fragments[i] = new Fragment(scans[i].getCanonicalName(), tablePath, 0, 0);
} else {
tablePath = tableDesc.getPath();
- stats[i] = tableDesc.getMeta().getStat();
- fragments[i] = storageManager.getSplits(scans[i].getCanonicalName(),
- tableDesc.getMeta(), tablePath).get(0);
+ stats[i] = tableDesc.getStats();
+ fragments[i] = storageManager.getSplits(scans[i].getCanonicalName(), tableDesc.getMeta(), tableDesc.getSchema(),
+ tablePath).get(0);
}
}
@@ -212,7 +210,8 @@ public class Repartitioner {
meta = desc.getMeta();
FileSystem fs = inputPath.getFileSystem(subQuery.getContext().getConf());
- List<Fragment> fragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, inputPath);
+ List<Fragment> fragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, desc.getSchema(),
+ inputPath);
QueryUnit queryUnit;
List<QueryUnit> queryUnits = new ArrayList<QueryUnit>();
@@ -310,7 +309,7 @@ public class Repartitioner {
SubQuery childSubQuery, DataChannel channel, int maxNum)
throws InternalException {
ExecutionBlock execBlock = subQuery.getBlock();
- TableStat stat = childSubQuery.getTableStat();
+ TableStats stat = childSubQuery.getTableStat();
if (stat.getNumRows() == 0) {
return new QueryUnit[0];
}
@@ -343,8 +342,7 @@ public class Repartitioner {
" sub ranges (total units: " + determinedTaskNum + ")");
TupleRange [] ranges = partitioner.partition(determinedTaskNum);
- Fragment dummyFragment = new Fragment(scan.getTableName(), tablePath,
- CatalogUtil.newTableMeta(scan.getInSchema(), StoreType.CSV),0, 0);
+ Fragment dummyFragment = new Fragment(scan.getTableName(), tablePath, 0, 0);
List<String> basicFetchURIs = new ArrayList<String>();
@@ -423,13 +421,13 @@ public class Repartitioner {
SubQuery childSubQuery, DataChannel channel, int maxNum) {
ExecutionBlock execBlock = subQuery.getBlock();
- List<TableStat> tableStats = new ArrayList<TableStat>();
+ List<TableStats> tableStatses = new ArrayList<TableStats>();
List<ExecutionBlock> childBlocks = masterPlan.getChilds(subQuery.getId());
for (ExecutionBlock childBlock : childBlocks) {
SubQuery childExecSM = subQuery.getContext().getSubQuery(childBlock.getId());
- tableStats.add(childExecSM.getTableStat());
+ tableStatses.add(childExecSM.getTableStat());
}
- TableStat totalStat = StatisticsUtil.computeStatFromUnionBlock(tableStats);
+ TableStats totalStat = StatisticsUtil.computeStatFromUnionBlock(tableStatses);
if (totalStat.getNumRows() == 0) {
return new QueryUnit[0];
@@ -440,8 +438,7 @@ public class Repartitioner {
tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName());
- Fragment frag = new Fragment(scan.getCanonicalName(), tablePath,
- CatalogUtil.newTableMeta(scan.getInSchema(), StoreType.CSV), 0, 0);
+ Fragment frag = new Fragment(scan.getCanonicalName(), tablePath, 0, 0);
Map<String, List<IntermediateEntry>> hashedByHost;
[3/5] TAJO-287: Refactor TableDesc, TableMeta, and Fragment. (hyunsik)
Posted by hy...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index e7cb7a0..0f1cbde 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -30,26 +30,23 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.*;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.QueryUnitId;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Options;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.ColumnStat;
+import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.catalog.statistics.StatisticsUtil;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.PlannerUtil;
+import org.apache.tajo.engine.planner.global.DataChannel;
+import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.planner.logical.GroupbyNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.planner.logical.ScanNode;
import org.apache.tajo.engine.planner.logical.StoreTableNode;
-import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.master.TaskRunnerGroupEvent;
import org.apache.tajo.master.TaskRunnerGroupEvent.EventType;
import org.apache.tajo.master.TaskScheduler;
@@ -79,8 +76,9 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
private MasterPlan masterPlan;
private ExecutionBlock block;
private int priority;
+ private Schema schema;
private TableMeta meta;
- private TableStat statistics;
+ private TableStats statistics;
private EventHandler eventHandler;
private final AbstractStorageManager sm;
private TaskSchedulerImpl taskScheduler;
@@ -264,12 +262,15 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
return tasks.get(qid);
}
- @SuppressWarnings("UnusedDeclaration")
+ public Schema getSchema() {
+ return schema;
+ }
+
public TableMeta getTableMeta() {
return meta;
}
- public TableStat getTableStat() {
+ public TableStats getTableStat() {
return statistics;
}
@@ -306,12 +307,12 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
}
}
- public static TableStat computeStatFromUnionBlock(SubQuery subQuery) {
- TableStat stat = new TableStat();
- TableStat childStat;
+ public static TableStats computeStatFromUnionBlock(SubQuery subQuery) {
+ TableStats stat = new TableStats();
+ TableStats childStat;
long avgRows = 0, numBytes = 0, numRows = 0;
int numBlocks = 0, numPartitions = 0;
- List<ColumnStat> columnStats = Lists.newArrayList();
+ List<ColumnStats> columnStatses = Lists.newArrayList();
MasterPlan masterPlan = subQuery.getMasterPlan();
Iterator<ExecutionBlock> it = masterPlan.getChilds(subQuery.getBlock()).iterator();
@@ -320,14 +321,14 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
SubQuery childSubQuery = subQuery.context.getSubQuery(block.getId());
childStat = childSubQuery.getTableStat();
avgRows += childStat.getAvgRows();
- columnStats.addAll(childStat.getColumnStats());
+ columnStatses.addAll(childStat.getColumnStats());
numBlocks += childStat.getNumBlocks();
numBytes += childStat.getNumBytes();
numPartitions += childStat.getNumPartitions();
numRows += childStat.getNumRows();
}
- stat.setColumnStats(columnStats);
+ stat.setColumnStats(columnStatses);
stat.setNumBlocks(numBlocks);
stat.setNumBytes(numBytes);
stat.setNumPartitions(numPartitions);
@@ -336,13 +337,13 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
return stat;
}
- private TableStat computeStatFromTasks() {
- List<TableStat> stats = Lists.newArrayList();
+ private TableStats computeStatFromTasks() {
+ List<TableStats> stats = Lists.newArrayList();
for (QueryUnit unit : getQueryUnits()) {
stats.add(unit.getStats());
}
- TableStat tableStat = StatisticsUtil.aggregateTableStat(stats);
- return tableStat;
+ TableStats tableStats = StatisticsUtil.aggregateTableStat(stats);
+ return tableStats;
}
private void stopScheduler() {
@@ -360,11 +361,11 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
}
private void finish() {
- TableStat stat;
+ TableStats stats;
if (block.hasUnion()) {
- stat = computeStatFromUnionBlock(this);
+ stats = computeStatFromUnionBlock(this);
} else {
- stat = computeStatFromTasks();
+ stats = computeStatFromTasks();
}
DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0);
@@ -376,12 +377,12 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
if (storeTableNode != null) {
storeType = storeTableNode.getStorageType();
}
- meta = CatalogUtil.newTableMeta(channel.getSchema(), storeType, new Options());
- meta.setStat(stat);
- statistics = stat;
+ schema = channel.getSchema();
+ meta = CatalogUtil.newTableMeta(storeType, new Options());
+ statistics = stats;
setFinishTime();
- eventHandler.handle(new SubQuerySucceeEvent(getId(), meta));
+ eventHandler.handle(new SubQuerySucceeEvent(getId()));
}
@Override
@@ -592,7 +593,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
Map<String, TableDesc> tableMap = context.getTableDescMap();
if (masterPlan.isLeaf(execBlock)) {
ScanNode outerScan = execBlock.getScanNodes()[0];
- TableStat stat = tableMap.get(outerScan.getCanonicalName()).getMeta().getStat();
+ TableStats stat = tableMap.get(outerScan.getCanonicalName()).getStats();
return stat.getNumBytes();
} else {
long aggregatedVolume = 0;
@@ -641,7 +642,8 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
meta = desc.getMeta();
// TODO - should be change the inner directory
- List<Fragment> fragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, inputPath);
+ List<Fragment> fragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, desc.getSchema(),
+ inputPath);
QueryUnit queryUnit;
List<QueryUnit> queryUnits = new ArrayList<QueryUnit>();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
index 2ba563c..c95ff14 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
@@ -13,7 +13,6 @@ import org.apache.tajo.ipc.ClientProtos;
import org.apache.tajo.util.JSPUtil;
import org.codehaus.jackson.map.DeserializationConfig;
import org.codehaus.jackson.map.ObjectMapper;
-import org.glassfish.grizzly.threadpool.FixedThreadPool;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
@@ -31,7 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/Task.java
index e77086d..cca7242 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/Task.java
@@ -19,6 +19,7 @@
package org.apache.tajo.worker;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,14 +32,13 @@ import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoProtos.TaskAttemptState;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.json.CoreGsonHelper;
import org.apache.tajo.engine.planner.PlannerUtil;
-import org.apache.tajo.engine.planner.logical.LogicalNode;
-import org.apache.tajo.engine.planner.logical.SortNode;
-import org.apache.tajo.engine.planner.logical.StoreTableNode;
+import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.planner.physical.PhysicalExec;
import org.apache.tajo.ipc.QueryMasterProtocol;
import org.apache.tajo.ipc.QueryMasterProtocol.*;
@@ -55,10 +55,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.text.NumberFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -78,6 +75,7 @@ public class Task {
private final TaskAttemptContext context;
private List<Fetcher> fetcherRunners;
private final LogicalNode plan;
+ private final Map<String, TableDesc> descs = Maps.newHashMap();
private PhysicalExec executor;
private boolean interQuery;
private boolean killed = false;
@@ -150,6 +148,12 @@ public class Task {
this.context.setEnforcer(request.getEnforcer());
plan = CoreGsonHelper.fromJson(request.getSerializedData(), LogicalNode.class);
+ LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN);
+ for (LogicalNode node : scanNode) {
+ ScanNode scan = (ScanNode)node;
+ descs.put(scan.getCanonicalName(), scan.getTableDesc());
+ }
+
interQuery = request.getProto().getInterQuery();
if (interQuery) {
context.setInterQuery();
@@ -179,7 +183,8 @@ public class Task {
LOG.info("* Fragments (num: " + request.getFragments().size() + ")");
for (Fragment f: request.getFragments()) {
- LOG.info("Table Id:" + f.getName() + ", path:" + f.getPath() + "(" + f.getMeta().getStoreType() + "), " +
+ TableDesc desc = descs.get(f.getName());
+ LOG.info("Table Id:" + f.getName() + ", path:" + desc.getPath() + "(" + desc.getMeta().getStoreType() + "), " +
"(start:" + f.getStartOffset() + ", length: " + f.getLength() + ")");
}
LOG.info("* Fetches (total:" + request.getFetches().size() + ") :");
@@ -328,7 +333,7 @@ public class Task {
if (context.hasResultStats()) {
builder.setResultStats(context.getResultStats().getProto());
} else {
- builder.setResultStats(new TableStat().getProto());
+ builder.setResultStats(new TableStats().getProto());
}
Iterator<Entry<Integer,String>> it = context.getRepartitions();
@@ -350,8 +355,7 @@ public class Task {
Collection<String> inputs = Lists.newArrayList(context.getInputTables());
for (String inputTable: inputs) {
File tableDir = new File(context.getFetchIn(), inputTable);
- Fragment [] frags = localizeFetchedData(tableDir, inputTable,
- context.getTable(inputTable).getMeta());
+ Fragment [] frags = localizeFetchedData(tableDir, inputTable, descs.get(inputTable).getMeta());
context.changeFragment(inputTable, frags);
}
}
@@ -461,7 +465,7 @@ public class Task {
if (f.getLen() == 0) {
continue;
}
- tablet = new Fragment(name, f.getPath(), meta, 0l, f.getLen());
+ tablet = new Fragment(name, f.getPath(), 0l, f.getLen());
listTablets.add(tablet);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
index 906c4d6..f05e1a1 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
@@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.TajoProtos.TaskAttemptState;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
@@ -46,7 +46,7 @@ public class TaskAttemptContext {
private final Map<String, List<Fragment>> fragmentMap = new HashMap<String, List<Fragment>>();
private TaskAttemptState state;
- private TableStat resultStats;
+ private TableStats resultStats;
private QueryUnitAttemptId queryId;
private final Path workDir;
private boolean needFetch = false;
@@ -115,11 +115,11 @@ public class TaskAttemptContext {
return resultStats != null;
}
- public void setResultStats(TableStat stats) {
+ public void setResultStats(TableStats stats) {
this.resultStats = stats;
}
- public TableStat getResultStats() {
+ public TableStats getResultStats() {
return this.resultStats;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/proto/ClientProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/proto/ClientProtos.proto b/tajo-core/tajo-core-backend/src/main/proto/ClientProtos.proto
index 5a70f91..89c40c8 100644
--- a/tajo-core/tajo-core-backend/src/main/proto/ClientProtos.proto
+++ b/tajo-core/tajo-core-backend/src/main/proto/ClientProtos.proto
@@ -121,8 +121,9 @@ message GetTableDescRequest {
message CreateTableRequest {
required string name = 1;
- required string path = 2;
+ required SchemaProto schema = 2;
required TableProto meta = 3;
+ required string path = 4;
}
message AttachTableRequest {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/proto/TajoMasterClientProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/proto/TajoMasterClientProtocol.proto b/tajo-core/tajo-core-backend/src/main/proto/TajoMasterClientProtocol.proto
index 17b237b..b70efc6 100644
--- a/tajo-core/tajo-core-backend/src/main/proto/TajoMasterClientProtocol.proto
+++ b/tajo-core/tajo-core-backend/src/main/proto/TajoMasterClientProtocol.proto
@@ -43,7 +43,6 @@ service TajoMasterClientProtocolService {
rpc getTableDesc(GetTableDescRequest) returns (TableResponse);
rpc createExternalTable(CreateTableRequest) returns (TableResponse);
rpc dropTable(StringProto) returns (BoolProto);
- rpc attachTable(AttachTableRequest) returns (TableResponse);
rpc detachTable(StringProto) returns (BoolProto);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/tajo-core-backend/src/main/proto/TajoWorkerProtocol.proto
index 109dfa9..987af25 100644
--- a/tajo-core/tajo-core-backend/src/main/proto/TajoWorkerProtocol.proto
+++ b/tajo-core/tajo-core-backend/src/main/proto/TajoWorkerProtocol.proto
@@ -34,14 +34,14 @@ message TaskStatusProto {
required float progress = 3;
required TaskAttemptState state = 4;
optional StatSetProto stats = 5;
- optional TableStatProto resultStats = 6;
+ optional TableStatsProto resultStats = 6;
repeated Partition partitions = 7;
}
message TaskCompletionReport {
required QueryUnitAttemptIdProto id = 1;
optional StatSetProto stats = 2;
- optional TableStatProto resultStats = 3;
+ optional TableStatsProto resultStats = 3;
repeated Partition partitions = 4;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/BackendTestingUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/BackendTestingUtil.java
index 61a8958..8f7189f 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/BackendTestingUtil.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/BackendTestingUtil.java
@@ -33,7 +33,6 @@ import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.storage.*;
-import org.apache.tajo.util.FileUtil;
import java.io.IOException;
@@ -46,7 +45,7 @@ public class BackendTestingUtil {
mockupSchema = new Schema();
mockupSchema.addColumn("deptname", Type.TEXT);
mockupSchema.addColumn("score", Type.INT4);
- mockupMeta = CatalogUtil.newTableMeta(mockupSchema, StoreType.CSV);
+ mockupMeta = CatalogUtil.newTableMeta(StoreType.CSV);
}
public static void writeTmpTable(TajoConf conf, Path tablePath)
@@ -62,7 +61,7 @@ public class BackendTestingUtil {
}
fs.mkdirs(tablePath);
- appender = sm.getAppender(mockupMeta, filePath);
+ appender = sm.getAppender(mockupMeta, mockupSchema, filePath);
appender.init();
int deptSize = 10000;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
index 178e8b3..fa67eb6 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -81,9 +81,8 @@ public class LocalTajoTestingUtility {
fs.mkdirs(tablePath);
Path dfsPath = new Path(tablePath, localPath.getName());
fs.copyFromLocalFile(localPath, dfsPath);
- TableMeta meta = CatalogUtil.newTableMeta(schemas[i],
- CatalogProtos.StoreType.CSV, option);
- client.createExternalTable(names[i], tablePath, meta);
+ TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV, option);
+ client.createExternalTable(names[i], schemas[i], tablePath, meta);
}
LOG.info("===================================================");
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
index e348f2f..116e282 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -501,32 +501,6 @@ public class TajoTestingCluster {
LOG.info("Minicluster is down");
}
- public static ResultSet runInLocal(String[] tableNames,
- Schema[] schemas,
- Options option,
- String[][] tables,
- String query) throws Exception {
- TajoTestingCluster util = new TajoTestingCluster();
- util.startMiniClusterInLocal(1);
- TajoConf conf = util.getConfiguration();
- TajoClient client = new TajoClient(conf);
-
- File tmpDir = util.setupClusterTestBuildDir();
- for (int i = 0; i < tableNames.length; i++) {
- File tableDir = new File(tmpDir,tableNames[i]);
- tableDir.mkdirs();
- File tableFile = new File(tableDir, tableNames[i]);
- writeLines(tableFile, tables[i]);
- TableMeta meta = CatalogUtil
- .newTableMeta(schemas[i], CatalogProtos.StoreType.CSV, option);
- client.createExternalTable(tableNames[i], new Path(tableDir.getAbsolutePath()), meta);
- }
- Thread.sleep(1000);
- ResultSet res = client.executeQueryAndGetResult(query);
- util.shutdownMiniCluster();
- return res;
- }
-
public static ResultSet run(String[] names,
Schema[] schemas,
Options option,
@@ -556,9 +530,8 @@ public class TajoTestingCluster {
out.write((tables[i][j]+"\n").getBytes());
}
out.close();
- TableMeta meta = CatalogUtil.newTableMeta(schemas[i],
- CatalogProtos.StoreType.CSV, option);
- client.createExternalTable(names[i], tablePath, meta);
+ TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV, option);
+ client.createExternalTable(names[i], schemas[i], tablePath, meta);
}
Thread.sleep(1000);
ResultSet res = client.executeQueryAndGetResult(query);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
index 6123f59..f91e8ec 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
@@ -37,11 +37,11 @@ public class TestDDLBuilder {
Schema schema = new Schema();
schema.addColumn("name", TajoDataTypes.Type.BLOB);
schema.addColumn("addr", TajoDataTypes.Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(schema, CatalogProtos.StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
meta.putOption("csv.delimiter", "|");
meta.putOption(TableMeta.COMPRESSION_CODEC, GzipCodec.class.getName());
- TableDesc desc = new TableDescImpl("table1", meta, new Path("/table1"));
+ TableDesc desc = new TableDesc("table1", schema, meta, new Path("/table1"));
assertEquals(FileUtil.readTextFile(new File("src/test/results/testBuildDDL.result")), DDLBuilder.buildDDL(desc));
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 09d8ceb..bd62f40 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -89,7 +89,7 @@ public class TestTajoClient {
assertFalse(client.existTable(tableName));
- client.createExternalTable(tableName, tablePath, BackendTestingUtil.mockupMeta);
+ client.createExternalTable(tableName, BackendTestingUtil.mockupSchema, tablePath, BackendTestingUtil.mockupMeta);
assertTrue(client.existTable(tableName));
client.dropTable(tableName);
assertFalse(client.existTable(tableName));
@@ -183,12 +183,12 @@ public class TestTajoClient {
assertNotNull(tablePath);
assertFalse(client.existTable(tableName1));
- client.createExternalTable("table3", tablePath, BackendTestingUtil.mockupMeta);
+ client.createExternalTable("table3", BackendTestingUtil.mockupSchema, tablePath, BackendTestingUtil.mockupMeta);
assertTrue(client.existTable(tableName1));
TableDesc desc = client.getTableDesc(tableName1);
assertNotNull(desc);
assertEquals(tableName1, desc.getName());
- assertTrue(desc.getMeta().getStat().getNumBytes() > 0);
+ assertTrue(desc.getStats().getNumBytes() > 0);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 95f024c..c35c786 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -114,7 +114,7 @@ public class ExprTestBase {
vtuple.put(i, lazyTuple.get(i));
}
}
- cat.addTable(new TableDescImpl(tableName, inputSchema, CatalogProtos.StoreType.CSV, new Options(), CommonTestingUtil.getTestDir()));
+ cat.addTable(new TableDesc(tableName, inputSchema, CatalogProtos.StoreType.CSV, new Options(), CommonTestingUtil.getTestDir()));
}
Target [] targets = null;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
index 1304de5..b99dcce 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java
@@ -65,8 +65,8 @@ public class TestEvalTree {
schema.addColumn("score", INT4);
schema.addColumn("age", INT4);
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- TableDesc desc = new TableDescImpl("people", meta, CommonTestingUtil.getTestDir());
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc desc = new TableDesc("people", schema, meta, CommonTestingUtil.getTestDir());
cat.addTable(desc);
FunctionDesc funcMeta = new FunctionDesc("test_sum", TestSum.class, FunctionType.GENERAL,
@@ -159,7 +159,7 @@ public class TestEvalTree {
EvalNode expr;
- Schema peopleSchema = cat.getTableDesc("people").getMeta().getSchema();
+ Schema peopleSchema = cat.getTableDesc("people").getSchema();
EvalContext evalCtx;
expr = getRootSelection(QUERIES[0]);
evalCtx = expr.newContext();
@@ -609,7 +609,7 @@ public class TestEvalTree {
assertTrue(not.terminate(evalCtx).asBool());
// Evaluation Test
- Schema peopleSchema = cat.getTableDesc("people").getMeta().getSchema();
+ Schema peopleSchema = cat.getTableDesc("people").getSchema();
expr = getRootSelection(NOT[0]);
evalCtx = expr.newContext();
expr.eval(evalCtx, peopleSchema, tuples[0]);
@@ -631,7 +631,7 @@ public class TestEvalTree {
public final void testLike() {
EvalNode expr;
- Schema peopleSchema = cat.getTableDesc("people").getMeta().getSchema();
+ Schema peopleSchema = cat.getTableDesc("people").getSchema();
// prefix
expr = getRootSelection(LIKE[0]);
EvalContext evalCtx = expr.newContext();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
index b4edf6a..1850081 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
@@ -72,8 +72,8 @@ public class TestEvalTreeUtil {
schema.addColumn("score", TajoDataTypes.Type.INT4);
schema.addColumn("age", TajoDataTypes.Type.INT4);
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- TableDesc desc = new TableDescImpl("people", meta, CommonTestingUtil.getTestDir());
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc desc = new TableDesc("people", schema, meta, CommonTestingUtil.getTestDir());
catalog.addTable(desc);
FunctionDesc funcMeta = new FunctionDesc("test_sum", TestSum.class,
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java
index 1a2b360..596ce1e 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java
@@ -47,12 +47,12 @@ public class TestLogicalNode {
schema.addColumn("age", Type.INT2);
GroupbyNode groupbyNode = new GroupbyNode(0, new Column[]{schema.getColumn(1), schema.getColumn(2)});
ScanNode scanNode = new ScanNode(0,
- CatalogUtil.newTableDesc("in", CatalogUtil.newTableMeta(schema, StoreType.CSV), new Path("in")));
+ CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));
GroupbyNode groupbyNode2 = new GroupbyNode(0, new Column[]{schema.getColumn(1), schema.getColumn(2)});
JoinNode joinNode = new JoinNode(0);
ScanNode scanNode2 = new ScanNode(0,
- CatalogUtil.newTableDesc("in2", CatalogUtil.newTableMeta(schema, StoreType.CSV), new Path("in2")));
+ CatalogUtil.newTableDesc("in2", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in2")));
groupbyNode.setChild(scanNode);
groupbyNode2.setChild(joinNode);
@@ -63,7 +63,7 @@ public class TestLogicalNode {
assertFalse(groupbyNode.deepEquals(groupbyNode2));
ScanNode scanNode3 = new ScanNode(0,
- CatalogUtil.newTableDesc("in", CatalogUtil.newTableMeta(schema, StoreType.CSV), new Path("in")));
+ CatalogUtil.newTableDesc("in", schema, CatalogUtil.newTableMeta(StoreType.CSV), new Path("in")));
groupbyNode2.setChild(scanNode3);
assertTrue(groupbyNode.equals(groupbyNode2));
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
index 8b25cf6..50bc8dc 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
@@ -66,14 +66,14 @@ public class TestLogicalOptimizer {
schema3.addColumn("score", Type.INT4);
schema3.addColumn("phone", Type.INT4);
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- TableDesc people = new TableDescImpl("employee", meta, CommonTestingUtil.getTestDir());
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc people = new TableDesc("employee", schema, meta, CommonTestingUtil.getTestDir());
catalog.addTable(people);
- TableDesc student = new TableDescImpl("dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
+ TableDesc student = new TableDesc("dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
catalog.addTable(student);
- TableDesc score = new TableDescImpl("score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
+ TableDesc score = new TableDesc("score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
catalog.addTable(score);
FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.GENERAL,
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
index d830591..025c84b 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java
@@ -22,7 +22,7 @@ import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.benchmark.TPCH;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph;
import org.apache.tajo.master.TajoMaster;
@@ -65,11 +65,12 @@ public class TestLogicalPlan {
tpch.loadOutSchema();
for (int i = 0; i < tpchTables.length; i++) {
- TableMeta m = CatalogUtil.newTableMeta(tpch.getSchema(tpchTables[i]), CatalogProtos.StoreType.CSV);
- TableStat stat = new TableStat();
- stat.setNumBytes(tableVolumns[i]);
- m.setStat(stat);
- TableDesc d = CatalogUtil.newTableDesc(tpchTables[i], m, CommonTestingUtil.getTestDir());
+ TableMeta m = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
+ TableStats stats = new TableStats();
+ stats.setNumBytes(tableVolumns[i]);
+ TableDesc d = CatalogUtil.newTableDesc(tpchTables[i], tpch.getSchema(tpchTables[i]), m,
+ CommonTestingUtil.getTestDir());
+ d.setStats(stats);
catalog.addTable(d);
}
planner = new LogicalPlanner(catalog);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index 9e4727b..b3aaf32 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -75,14 +75,14 @@ public class TestLogicalPlanner {
schema3.addColumn("deptname", Type.TEXT);
schema3.addColumn("score", Type.INT4);
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- TableDesc people = new TableDescImpl("employee", meta, CommonTestingUtil.getTestDir());
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc people = new TableDesc("employee", schema, meta, CommonTestingUtil.getTestDir());
catalog.addTable(people);
- TableDesc student = new TableDescImpl("dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
+ TableDesc student = new TableDesc("dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
catalog.addTable(student);
- TableDesc score = new TableDescImpl("score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
+ TableDesc score = new TableDesc("score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
catalog.addTable(score);
FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.AGGREGATION,
@@ -98,8 +98,8 @@ public class TestLogicalPlanner {
tpch.loadSchemas();
tpch.loadOutSchema();
for (String table : tpchTables) {
- TableMeta m = CatalogUtil.newTableMeta(tpch.getSchema(table), StoreType.CSV);
- TableDesc d = CatalogUtil.newTableDesc(table, m, CommonTestingUtil.getTestDir());
+ TableMeta m = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc d = CatalogUtil.newTableDesc(table, tpch.getSchema(table), m, CommonTestingUtil.getTestDir());
catalog.addTable(d);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
index b2727b2..b456cf9 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
@@ -66,14 +66,14 @@ public class TestPlannerUtil {
schema3.addColumn("deptName", Type.TEXT);
schema3.addColumn("score", CatalogUtil.newSimpleDataType(Type.INT4));
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- TableDesc people = new TableDescImpl("employee", meta, CommonTestingUtil.getTestDir());
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableDesc people = new TableDesc("employee", schema, meta, CommonTestingUtil.getTestDir());
catalog.addTable(people);
- TableDesc student = new TableDescImpl("dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
+ TableDesc student = new TableDesc("dept", schema2, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
catalog.addTable(student);
- TableDesc score = new TableDescImpl("score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
+ TableDesc score = new TableDesc("score", schema3, StoreType.CSV, new Options(), CommonTestingUtil.getTestDir());
catalog.addTable(score);
FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.AGGREGATION,
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
index 555b698..4d2bd6c 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
@@ -78,11 +78,11 @@ public class TestBNLJoinExec {
schema.addColumn("memId", Type.INT4);
schema.addColumn("deptName", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < OUTER_TUPLE_NUM; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createInt4(i), DatumFactory.createInt4(10 + i),
@@ -91,7 +91,7 @@ public class TestBNLJoinExec {
}
appender.flush();
appender.close();
- employee = CatalogUtil.newTableDesc("employee", employeeMeta, employeePath);
+ employee = CatalogUtil.newTableDesc("employee", schema, employeeMeta, employeePath);
catalog.addTable(employee);
Schema peopleSchema = new Schema();
@@ -99,11 +99,11 @@ public class TestBNLJoinExec {
peopleSchema.addColumn("fk_memId", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(peopleSchema, StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path peoplePath = new Path(testDir, "people.csv");
- appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peoplePath);
+ appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath);
appender.init();
- tuple = new VTuple(peopleMeta.getSchema().getColumnNum());
+ tuple = new VTuple(peopleSchema.getColumnNum());
for (int i = 1; i < INNER_TUPLE_NUM; i += 2) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createInt4(10 + i),
@@ -114,7 +114,7 @@ public class TestBNLJoinExec {
appender.flush();
appender.close();
- people = CatalogUtil.newTableDesc("people", peopleMeta, peoplePath);
+ people = CatalogUtil.newTableDesc("people", peopleSchema, peopleMeta, peoplePath);
catalog.addTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
index 8945832..615befa 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
@@ -103,14 +103,15 @@ public class TestBSTIndexExec {
writer.open();
long offset;
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ meta = CatalogUtil.newTableMeta(StoreType.CSV);
tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv");
fs = tablePath.getFileSystem(conf);
fs.mkdirs(tablePath.getParent());
- FileAppender appender = (FileAppender)StorageManagerFactory.getStorageManager(conf).getAppender(meta, tablePath);
+ FileAppender appender = (FileAppender)StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema,
+ tablePath);
appender.init();
- Tuple tuple = new VTuple(meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < 10000; i++) {
Tuple key = new VTuple(this.idxSchema.getColumnNum());
@@ -134,8 +135,7 @@ public class TestBSTIndexExec {
appender.close();
writer.close();
- TableDesc desc = new TableDescImpl("employee", meta,
- sm.getTablePath("employee"));
+ TableDesc desc = new TableDesc("employee", schema, meta, sm.getTablePath("employee"));
catalog.addTable(desc);
analyzer = new SQLAnalyzer();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index febb315..282f5be 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -75,12 +75,12 @@ public class TestExternalSortExec {
schema.addColumn("empId", Type.INT4);
schema.addColumn("deptName", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath);
appender.enableStats();
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < numTuple; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(rnd.nextInt(50)),
DatumFactory.createInt4(rnd.nextInt(100)),
@@ -92,7 +92,7 @@ public class TestExternalSortExec {
System.out.println("Total Rows: " + appender.getStats().getNumRows());
- employee = new TableDescImpl("employee", employeeMeta, employeePath);
+ employee = new TableDesc("employee", schema, employeeMeta, employeePath);
catalog.addTable(employee);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index f293f70..c7fec48 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -91,11 +91,11 @@ public class TestFullOuterHashJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(dep3Schema, StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep3Path = new Path(testDir, "dep3.csv");
- Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Path);
+ Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
- Tuple tuple = new VTuple(dep3Meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(dep3Schema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -105,7 +105,7 @@ public class TestFullOuterHashJoinExec {
appender1.flush();
appender1.close();
- dep3 = CatalogUtil.newTableDesc("dep3", dep3Meta, dep3Path);
+ dep3 = CatalogUtil.newTableDesc("dep3", dep3Schema, dep3Meta, dep3Path);
catalog.addTable(dep3);
//----------------- job3 ------------------------------
@@ -120,12 +120,11 @@ public class TestFullOuterHashJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(job3Schema,
- StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path job3Path = new Path(testDir, "job3.csv");
- Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Path);
+ Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
- Tuple tuple2 = new VTuple(job3Meta.getSchema().getColumnNum());
+ Tuple tuple2 = new VTuple(job3Schema.getColumnNum());
for (int i = 1; i < 4; i++) {
int x = 100 + i;
tuple2.put(new Datum[] { DatumFactory.createInt4(100 + i),
@@ -135,7 +134,7 @@ public class TestFullOuterHashJoinExec {
appender2.flush();
appender2.close();
- job3 = CatalogUtil.newTableDesc("job3", job3Meta, job3Path);
+ job3 = CatalogUtil.newTableDesc("job3", job3Schema, job3Meta, job3Path);
catalog.addTable(job3);
@@ -160,11 +159,11 @@ public class TestFullOuterHashJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(emp3Schema, StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path emp3Path = new Path(testDir, "emp3.csv");
- Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Path);
+ Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
- Tuple tuple3 = new VTuple(emp3Meta.getSchema().getColumnNum());
+ Tuple tuple3 = new VTuple(emp3Schema.getColumnNum());
for (int i = 1; i < 4; i += 2) {
int x = 10 + i;
@@ -199,7 +198,7 @@ public class TestFullOuterHashJoinExec {
appender3.flush();
appender3.close();
- emp3 = CatalogUtil.newTableDesc("emp3", emp3Meta, emp3Path);
+ emp3 = CatalogUtil.newTableDesc("emp3", emp3Schema, emp3Meta, emp3Path);
catalog.addTable(emp3);
//---------------------phone3 --------------------
@@ -212,15 +211,15 @@ public class TestFullOuterHashJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(phone3Schema,
- StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path phone3Path = new Path(testDir, "phone3.csv");
- Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Path);
+ Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema,
+ phone3Path);
appender5.init();
appender5.flush();
appender5.close();
- phone3 = CatalogUtil.newTableDesc("phone3", phone3Meta, phone3Path);
+ phone3 = CatalogUtil.newTableDesc("phone3", phone3Schema, phone3Meta, phone3Path);
catalog.addTable(phone3);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index 8d0466f..61802fd 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -94,11 +94,11 @@ public class TestFullOuterMergeJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(dep3Schema, StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep3Path = new Path(testDir, "dep3.csv");
- Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Path);
+ Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
- Tuple tuple = new VTuple(dep3Meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(dep3Schema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -108,7 +108,7 @@ public class TestFullOuterMergeJoinExec {
appender1.flush();
appender1.close();
- dep3 = CatalogUtil.newTableDesc("dep3", dep3Meta, dep3Path);
+ dep3 = CatalogUtil.newTableDesc("dep3", dep3Schema, dep3Meta, dep3Path);
catalog.addTable(dep3);
@@ -132,12 +132,11 @@ public class TestFullOuterMergeJoinExec {
dep4Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep4Meta = CatalogUtil.newTableMeta(dep4Schema,
- StoreType.CSV);
+ TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep4Path = new Path(testDir, "dep4.csv");
- Appender appender4 = StorageManagerFactory.getStorageManager(conf).getAppender(dep4Meta, dep4Path);
+ Appender appender4 = StorageManagerFactory.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path);
appender4.init();
- Tuple tuple4 = new VTuple(dep4Meta.getSchema().getColumnNum());
+ Tuple tuple4 = new VTuple(dep4Schema.getColumnNum());
for (int i = 0; i < 11; i++) {
tuple4.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -147,7 +146,7 @@ public class TestFullOuterMergeJoinExec {
appender4.flush();
appender4.close();
- dep4 = CatalogUtil.newTableDesc("dep4", dep4Meta, dep4Path);
+ dep4 = CatalogUtil.newTableDesc("dep4", dep4Schema, dep4Meta, dep4Path);
catalog.addTable(dep4);
@@ -164,12 +163,11 @@ public class TestFullOuterMergeJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(job3Schema,
- StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path job3Path = new Path(testDir, "job3.csv");
- Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Path);
+ Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
- Tuple tuple2 = new VTuple(job3Meta.getSchema().getColumnNum());
+ Tuple tuple2 = new VTuple(job3Schema.getColumnNum());
for (int i = 1; i < 4; i++) {
int x = 100 + i;
tuple2.put(new Datum[] { DatumFactory.createInt4(100 + i),
@@ -179,7 +177,7 @@ public class TestFullOuterMergeJoinExec {
appender2.flush();
appender2.close();
- job3 = CatalogUtil.newTableDesc("job3", job3Meta, job3Path);
+ job3 = CatalogUtil.newTableDesc("job3", job3Schema, job3Meta, job3Path);
catalog.addTable(job3);
@@ -204,11 +202,11 @@ public class TestFullOuterMergeJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(emp3Schema, StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path emp3Path = new Path(testDir, "emp3.csv");
- Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Path);
+ Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
- Tuple tuple3 = new VTuple(emp3Meta.getSchema().getColumnNum());
+ Tuple tuple3 = new VTuple(emp3Schema.getColumnNum());
for (int i = 1; i < 4; i += 2) {
int x = 10 + i;
@@ -243,7 +241,7 @@ public class TestFullOuterMergeJoinExec {
appender3.flush();
appender3.close();
- emp3 = CatalogUtil.newTableDesc("emp3", emp3Meta, emp3Path);
+ emp3 = CatalogUtil.newTableDesc("emp3", emp3Schema, emp3Meta, emp3Path);
catalog.addTable(emp3);
//---------------------phone3 --------------------
@@ -256,14 +254,14 @@ public class TestFullOuterMergeJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(phone3Schema,
- StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path phone3Path = new Path(testDir, "phone3.csv");
- Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Path);
+ Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema,
+ phone3Path);
appender5.init();
appender5.flush();
appender5.close();
- phone3 = CatalogUtil.newTableDesc("phone3", phone3Meta, phone3Path);
+ phone3 = CatalogUtil.newTableDesc("phone3", phone3Schema, phone3Meta, phone3Path);
catalog.addTable(phone3);
analyzer = new SQLAnalyzer();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index a91bbdd..e521a94 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -74,12 +74,12 @@ public class TestHashAntiJoinExec {
employeeSchema.addColumn("memId", Type.INT4);
employeeSchema.addColumn("deptName", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(employeeSchema,
- StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema,
+ employeePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(employeeSchema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] {
@@ -92,7 +92,7 @@ public class TestHashAntiJoinExec {
appender.flush();
appender.close();
- employee = CatalogUtil.newTableDesc("employee", employeeMeta, employeePath);
+ employee = CatalogUtil.newTableDesc("employee", employeeSchema, employeeMeta, employeePath);
catalog.addTable(employee);
Schema peopleSchema = new Schema();
@@ -100,11 +100,11 @@ public class TestHashAntiJoinExec {
peopleSchema.addColumn("fk_memId", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(peopleSchema, StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path peoplePath = new Path(testDir, "people.csv");
- appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peoplePath);
+ appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath);
appender.init();
- tuple = new VTuple(peopleMeta.getSchema().getColumnNum());
+ tuple = new VTuple(peopleSchema.getColumnNum());
for (int i = 1; i < 10; i += 2) {
tuple.put(new Datum[] {
DatumFactory.createInt4(i), // empid [1, 3, 5, 7, 9]
@@ -117,7 +117,7 @@ public class TestHashAntiJoinExec {
appender.flush();
appender.close();
- people = CatalogUtil.newTableDesc("people", peopleMeta, peoplePath);
+ people = CatalogUtil.newTableDesc("people", peopleSchema, peopleMeta, peoplePath);
catalog.addTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index 53ae60f..ba02c21 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -76,12 +76,12 @@ public class TestHashJoinExec {
employeeSchema.addColumn("memId", Type.INT4);
employeeSchema.addColumn("deptName", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(employeeSchema,
- StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema,
+ employeePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(employeeSchema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createInt4(i), DatumFactory.createInt4(10 + i),
@@ -91,7 +91,7 @@ public class TestHashJoinExec {
appender.flush();
appender.close();
- employee = CatalogUtil.newTableDesc("employee", employeeMeta, employeePath);
+ employee = CatalogUtil.newTableDesc("employee", employeeSchema, employeeMeta, employeePath);
catalog.addTable(employee);
Schema peopleSchema = new Schema();
@@ -99,11 +99,11 @@ public class TestHashJoinExec {
peopleSchema.addColumn("fk_memId", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(peopleSchema, StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path peoplePath = new Path(testDir, "people.csv");
- appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peoplePath);
+ appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath);
appender.init();
- tuple = new VTuple(peopleMeta.getSchema().getColumnNum());
+ tuple = new VTuple(peopleSchema.getColumnNum());
for (int i = 1; i < 10; i += 2) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createInt4(10 + i),
@@ -115,7 +115,7 @@ public class TestHashJoinExec {
appender.flush();
appender.close();
- people = CatalogUtil.newTableDesc("people", peopleMeta, peoplePath);
+ people = CatalogUtil.newTableDesc("people", peopleSchema, peopleMeta, peoplePath);
catalog.addTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index c5553f0..e4dcb95 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -74,12 +74,12 @@ public class TestHashSemiJoinExec {
employeeSchema.addColumn("memId", Type.INT4);
employeeSchema.addColumn("deptName", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(employeeSchema,
- StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema,
+ employeePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(employeeSchema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] {
@@ -92,7 +92,7 @@ public class TestHashSemiJoinExec {
appender.flush();
appender.close();
- employee = CatalogUtil.newTableDesc("employee", employeeMeta, employeePath);
+ employee = CatalogUtil.newTableDesc("employee", employeeSchema, employeeMeta, employeePath);
catalog.addTable(employee);
Schema peopleSchema = new Schema();
@@ -100,11 +100,11 @@ public class TestHashSemiJoinExec {
peopleSchema.addColumn("fk_memId", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(peopleSchema, StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path peoplePath = new Path(testDir, "people.csv");
- appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peoplePath);
+ appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath);
appender.init();
- tuple = new VTuple(peopleMeta.getSchema().getColumnNum());
+ tuple = new VTuple(peopleSchema.getColumnNum());
// make 27 tuples
for (int i = 1; i < 10; i += 2) {
// make three duplicated tuples for each tuples
@@ -121,7 +121,7 @@ public class TestHashSemiJoinExec {
appender.flush();
appender.close();
- people = CatalogUtil.newTableDesc("people", peopleMeta, peoplePath);
+ people = CatalogUtil.newTableDesc("people", peopleSchema, peopleMeta, peoplePath);
catalog.addTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index 0317a15..58f9b3b 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -91,12 +91,11 @@ public class TestLeftOuterHashJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(dep3Schema,
- StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep3Path = new Path(testDir, "dep3.csv");
- Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Path);
+ Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
- Tuple tuple = new VTuple(dep3Meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(dep3Schema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -106,7 +105,7 @@ public class TestLeftOuterHashJoinExec {
appender1.flush();
appender1.close();
- dep3 = CatalogUtil.newTableDesc("dep3", dep3Meta, dep3Path);
+ dep3 = CatalogUtil.newTableDesc("dep3", dep3Schema, dep3Meta, dep3Path);
catalog.addTable(dep3);
//----------------- job3 ------------------------------
@@ -121,12 +120,11 @@ public class TestLeftOuterHashJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(job3Schema,
- StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path job3Path = new Path(testDir, "job3.csv");
- Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Path);
+ Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
- Tuple tuple2 = new VTuple(job3Meta.getSchema().getColumnNum());
+ Tuple tuple2 = new VTuple(job3Schema.getColumnNum());
for (int i = 1; i < 4; i++) {
int x = 100 + i;
tuple2.put(new Datum[] { DatumFactory.createInt4(100 + i),
@@ -136,7 +134,7 @@ public class TestLeftOuterHashJoinExec {
appender2.flush();
appender2.close();
- job3 = CatalogUtil.newTableDesc("job3", job3Meta, job3Path);
+ job3 = CatalogUtil.newTableDesc("job3", job3Schema, job3Meta, job3Path);
catalog.addTable(job3);
@@ -161,11 +159,11 @@ public class TestLeftOuterHashJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(emp3Schema, StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path emp3Path = new Path(testDir, "emp3.csv");
- Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Path);
+ Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
- Tuple tuple3 = new VTuple(emp3Meta.getSchema().getColumnNum());
+ Tuple tuple3 = new VTuple(emp3Schema.getColumnNum());
for (int i = 1; i < 4; i += 2) {
int x = 10 + i;
@@ -200,7 +198,7 @@ public class TestLeftOuterHashJoinExec {
appender3.flush();
appender3.close();
- emp3 = CatalogUtil.newTableDesc("emp3", emp3Meta, emp3Path);
+ emp3 = CatalogUtil.newTableDesc("emp3", emp3Schema, emp3Meta, emp3Path);
catalog.addTable(emp3);
//---------------------phone3 --------------------
@@ -213,15 +211,15 @@ public class TestLeftOuterHashJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(phone3Schema,
- StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path phone3Path = new Path(testDir, "phone3.csv");
- Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Path);
+ Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema,
+ phone3Path);
appender5.init();
appender5.flush();
appender5.close();
- phone3 = CatalogUtil.newTableDesc("phone3", phone3Meta, phone3Path);
+ phone3 = CatalogUtil.newTableDesc("phone3", phone3Schema, phone3Meta, phone3Path);
catalog.addTable(phone3);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
index 32cf936..025af87 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
@@ -19,8 +19,8 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.worker.TaskAttemptContext;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
@@ -37,16 +37,15 @@ import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.storage.*;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.TUtil;
+import org.apache.tajo.worker.TaskAttemptContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.tajo.LocalTajoTestingUtility;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
public class TestLeftOuterNLJoinExec {
private TajoConf conf;
@@ -90,12 +89,11 @@ public class TestLeftOuterNLJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(dep3Schema,
- StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep3Path = new Path(testDir, "dep3.csv");
- Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Path);
+ Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
- Tuple tuple = new VTuple(dep3Meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(dep3Schema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -105,7 +103,7 @@ public class TestLeftOuterNLJoinExec {
appender1.flush();
appender1.close();
- dep3 = CatalogUtil.newTableDesc("dep3", dep3Meta, dep3Path);
+ dep3 = CatalogUtil.newTableDesc("dep3", dep3Schema, dep3Meta, dep3Path);
catalog.addTable(dep3);
//----------------- job3 ------------------------------
@@ -120,12 +118,11 @@ public class TestLeftOuterNLJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(job3Schema,
- StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path job3Path = new Path(testDir, "job3.csv");
- Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Path);
+ Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
- Tuple tuple2 = new VTuple(job3Meta.getSchema().getColumnNum());
+ Tuple tuple2 = new VTuple(job3Schema.getColumnNum());
for (int i = 1; i < 4; i++) {
int x = 100 + i;
tuple2.put(new Datum[] { DatumFactory.createInt4(100 + i),
@@ -135,7 +132,7 @@ public class TestLeftOuterNLJoinExec {
appender2.flush();
appender2.close();
- job3 = CatalogUtil.newTableDesc("job3", job3Meta, job3Path);
+ job3 = CatalogUtil.newTableDesc("job3", job3Schema, job3Meta, job3Path);
catalog.addTable(job3);
@@ -160,11 +157,11 @@ public class TestLeftOuterNLJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(emp3Schema, StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path emp3Path = new Path(testDir, "emp3.csv");
- Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Path);
+ Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
- Tuple tuple3 = new VTuple(emp3Meta.getSchema().getColumnNum());
+ Tuple tuple3 = new VTuple(emp3Schema.getColumnNum());
for (int i = 1; i < 4; i += 2) {
int x = 10 + i;
@@ -199,7 +196,7 @@ public class TestLeftOuterNLJoinExec {
appender3.flush();
appender3.close();
- emp3 = CatalogUtil.newTableDesc("emp3", emp3Meta, emp3Path);
+ emp3 = CatalogUtil.newTableDesc("emp3", emp3Schema, emp3Meta, emp3Path);
catalog.addTable(emp3);
// ---------------------phone3 --------------------
@@ -212,15 +209,15 @@ public class TestLeftOuterNLJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(phone3Schema,
- StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path phone3Path = new Path(testDir, "phone3.csv");
- Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Path);
+ Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema,
+ phone3Path);
appender5.init();
appender5.flush();
appender5.close();
- phone3 = CatalogUtil.newTableDesc("phone3", phone3Meta, phone3Path);
+ phone3 = CatalogUtil.newTableDesc("phone3", phone3Schema, phone3Meta, phone3Path);
catalog.addTable(phone3);
analyzer = new SQLAnalyzer();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index 1149c3f..5e72428 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -77,12 +77,12 @@ public class TestMergeJoinExec {
employeeSchema.addColumn("memId", Type.INT4);
employeeSchema.addColumn("deptName", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(employeeSchema,
- StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema,
+ employeePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(employeeSchema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createInt4(i), DatumFactory.createInt4(10 + i),
@@ -98,8 +98,7 @@ public class TestMergeJoinExec {
appender.flush();
appender.close();
- employee = CatalogUtil.newTableDesc("employee", employeeMeta,
- employeePath);
+ employee = CatalogUtil.newTableDesc("employee", employeeSchema, employeeMeta, employeePath);
catalog.addTable(employee);
Schema peopleSchema = new Schema();
@@ -107,11 +106,11 @@ public class TestMergeJoinExec {
peopleSchema.addColumn("fk_memId", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(peopleSchema, StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path peoplePath = new Path(testDir, "people.csv");
- appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peoplePath);
+ appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath);
appender.init();
- tuple = new VTuple(peopleMeta.getSchema().getColumnNum());
+ tuple = new VTuple(peopleSchema.getColumnNum());
for (int i = 1; i < 10; i += 2) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createInt4(10 + i),
@@ -130,7 +129,7 @@ public class TestMergeJoinExec {
appender.flush();
appender.close();
- people = CatalogUtil.newTableDesc("people", peopleMeta, peoplePath);
+ people = CatalogUtil.newTableDesc("people", peopleSchema, peopleMeta, peoplePath);
catalog.addTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
index d4498c5..3d356ee 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
@@ -78,11 +78,11 @@ public class TestNLJoinExec {
schema.addColumn("memId", Type.INT4);
schema.addColumn("deptName", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < 50; i++) {
tuple.put(new Datum[] {
DatumFactory.createInt4(i),
@@ -93,8 +93,7 @@ public class TestNLJoinExec {
}
appender.flush();
appender.close();
- employee = CatalogUtil.newTableDesc("employee", employeeMeta,
- employeePath);
+ employee = CatalogUtil.newTableDesc("employee", schema, employeeMeta, employeePath);
catalog.addTable(employee);
Schema peopleSchema = new Schema();
@@ -102,11 +101,11 @@ public class TestNLJoinExec {
peopleSchema.addColumn("fk_memId", Type.INT4);
peopleSchema.addColumn("name", Type.TEXT);
peopleSchema.addColumn("age", Type.INT4);
- TableMeta peopleMeta = CatalogUtil.newTableMeta(peopleSchema, StoreType.CSV);
+ TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path peoplePath = new Path(testDir, "people.csv");
- appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peoplePath);
+ appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath);
appender.init();
- tuple = new VTuple(peopleMeta.getSchema().getColumnNum());
+ tuple = new VTuple(peopleSchema.getColumnNum());
for (int i = 1; i < 50; i += 2) {
tuple.put(new Datum[] {
DatumFactory.createInt4(i),
@@ -118,8 +117,7 @@ public class TestNLJoinExec {
appender.flush();
appender.close();
- people = CatalogUtil.newTableDesc("people", peopleMeta,
- peoplePath);
+ people = CatalogUtil.newTableDesc("people", peopleSchema, peopleMeta, peoplePath);
catalog.addTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
[5/5] git commit: TAJO-287: Refactor TableDesc, TableMeta,
and Fragment. (hyunsik)
Posted by hy...@apache.org.
TAJO-287: Refactor TableDesc, TableMeta, and Fragment. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/7dc8de28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/7dc8de28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/7dc8de28
Branch: refs/heads/master
Commit: 7dc8de2833aec5b2d7b9b25d238775e74165243d
Parents: 5470bce
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sun Oct 27 15:40:22 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sun Oct 27 15:40:22 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 5 +-
.../org/apache/tajo/catalog/CatalogService.java | 4 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 60 +----
.../org/apache/tajo/catalog/DDLBuilder.java | 2 +-
.../java/org/apache/tajo/catalog/TableDesc.java | 150 +++++++++--
.../org/apache/tajo/catalog/TableDescImpl.java | 133 ----------
.../java/org/apache/tajo/catalog/TableMeta.java | 202 +++++++++++---
.../org/apache/tajo/catalog/TableMetaImpl.java | 263 -------------------
.../java/org/apache/tajo/catalog/TableUtil.java | 4 +-
.../tajo/catalog/json/CatalogGsonHelper.java | 2 -
.../tajo/catalog/json/TableDescAdapter.java | 49 ----
.../tajo/catalog/json/TableMetaAdapter.java | 21 +-
.../tajo/catalog/statistics/ColumnStat.java | 173 ------------
.../tajo/catalog/statistics/ColumnStats.java | 173 ++++++++++++
.../tajo/catalog/statistics/StatisticsUtil.java | 34 +--
.../tajo/catalog/statistics/TableStat.java | 204 --------------
.../tajo/catalog/statistics/TableStats.java | 208 +++++++++++++++
.../src/main/proto/CatalogProtos.proto | 23 +-
.../org/apache/tajo/catalog/TestTableDesc.java | 40 ++-
.../org/apache/tajo/catalog/TestTableInfo.java | 129 ---------
.../org/apache/tajo/catalog/TestTableMeta.java | 80 +-----
.../tajo/catalog/statistics/TestColumnStat.java | 16 +-
.../catalog/statistics/TestStatisticsUtil.java | 6 +-
.../tajo/catalog/statistics/TestTableStat.java | 14 +-
.../org/apache/tajo/catalog/CatalogServer.java | 17 +-
.../tajo/catalog/store/AbstractDBStore.java | 20 +-
.../apache/tajo/catalog/store/DerbyStore.java | 20 +-
.../org/apache/tajo/catalog/store/MemStore.java | 12 +-
.../org/apache/tajo/catalog/TestCatalog.java | 4 +-
.../org/apache/tajo/catalog/TestDBStore.java | 30 +--
.../java/org/apache/tajo/benchmark/TPCH.java | 4 +-
.../main/java/org/apache/tajo/cli/TajoCli.java | 14 +-
.../org/apache/tajo/client/ResultSetImpl.java | 10 +-
.../tajo/client/ResultSetMetaDataImpl.java | 23 +-
.../java/org/apache/tajo/client/TajoClient.java | 20 +-
.../apache/tajo/engine/json/CoreGsonHelper.java | 8 +-
.../tajo/engine/planner/LogicalPlanner.java | 10 +-
.../engine/planner/global/GlobalPlanner.java | 12 +-
.../join/GreedyHeuristicJoinOrderAlgorithm.java | 4 +-
.../planner/physical/BSTIndexScanExec.java | 2 +-
.../planner/physical/ExternalSortExec.java | 16 +-
.../planner/physical/IndexedStoreExec.java | 4 +-
.../planner/physical/PartitionedStoreExec.java | 12 +-
.../engine/planner/physical/SeqScanExec.java | 6 +-
.../engine/planner/physical/StoreTableExec.java | 16 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 8 +-
.../org/apache/tajo/master/GlobalEngine.java | 26 +-
.../tajo/master/TajoMasterClientService.java | 79 +-----
.../tajo/master/event/SubQuerySucceeEvent.java | 10 +-
.../apache/tajo/master/querymaster/Query.java | 19 +-
.../tajo/master/querymaster/QueryUnit.java | 9 +-
.../master/querymaster/QueryUnitAttempt.java | 4 +-
.../tajo/master/querymaster/Repartitioner.java | 35 ++-
.../tajo/master/querymaster/SubQuery.java | 62 ++---
.../tajo/webapp/QueryExecutorServlet.java | 2 -
.../main/java/org/apache/tajo/worker/Task.java | 30 ++-
.../apache/tajo/worker/TaskAttemptContext.java | 8 +-
.../src/main/proto/ClientProtos.proto | 3 +-
.../main/proto/TajoMasterClientProtocol.proto | 1 -
.../src/main/proto/TajoWorkerProtocol.proto | 4 +-
.../org/apache/tajo/BackendTestingUtil.java | 5 +-
.../apache/tajo/LocalTajoTestingUtility.java | 5 +-
.../org/apache/tajo/TajoTestingCluster.java | 31 +--
.../org/apache/tajo/client/TestDDLBuilder.java | 4 +-
.../org/apache/tajo/client/TestTajoClient.java | 6 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 2 +-
.../apache/tajo/engine/eval/TestEvalTree.java | 10 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 4 +-
.../tajo/engine/planner/TestLogicalNode.java | 6 +-
.../engine/planner/TestLogicalOptimizer.java | 8 +-
.../tajo/engine/planner/TestLogicalPlan.java | 13 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 12 +-
.../tajo/engine/planner/TestPlannerUtil.java | 8 +-
.../planner/physical/TestBNLJoinExec.java | 16 +-
.../planner/physical/TestBSTIndexExec.java | 10 +-
.../planner/physical/TestExternalSortExec.java | 8 +-
.../physical/TestFullOuterHashJoinExec.java | 33 ++-
.../physical/TestFullOuterMergeJoinExec.java | 42 ++-
.../planner/physical/TestHashAntiJoinExec.java | 18 +-
.../planner/physical/TestHashJoinExec.java | 18 +-
.../planner/physical/TestHashSemiJoinExec.java | 18 +-
.../physical/TestLeftOuterHashJoinExec.java | 34 ++-
.../physical/TestLeftOuterNLJoinExec.java | 39 ++-
.../planner/physical/TestMergeJoinExec.java | 19 +-
.../engine/planner/physical/TestNLJoinExec.java | 18 +-
.../planner/physical/TestPhysicalPlanner.java | 58 ++--
.../physical/TestRightOuterHashJoinExec.java | 31 +--
.../physical/TestRightOuterMergeJoinExec.java | 43 ++-
.../engine/planner/physical/TestSortExec.java | 8 +-
.../tajo/engine/query/TestInsertQuery.java | 20 +-
.../tajo/engine/query/TestResultSetImpl.java | 28 +-
.../tajo/engine/query/TestSelectQuery.java | 2 +-
.../tajo/master/TestExecutionBlockCursor.java | 8 +-
.../apache/tajo/master/TestGlobalPlanner.java | 13 +-
.../org/apache/tajo/storage/TestFragment.java | 40 +--
.../org/apache/tajo/storage/TestRowFile.java | 19 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 20 +-
.../tajo/storage/AbstractStorageManager.java | 77 ++++--
.../java/org/apache/tajo/storage/Appender.java | 4 +-
.../java/org/apache/tajo/storage/CSVFile.java | 23 +-
.../org/apache/tajo/storage/FileAppender.java | 4 +-
.../org/apache/tajo/storage/FileScanner.java | 4 +-
.../java/org/apache/tajo/storage/Fragment.java | 68 +----
.../org/apache/tajo/storage/MergeScanner.java | 12 +-
.../java/org/apache/tajo/storage/RawFile.java | 17 +-
.../java/org/apache/tajo/storage/RowFile.java | 17 +-
.../org/apache/tajo/storage/StorageManager.java | 28 +-
.../tajo/storage/StorageManagerFactory.java | 14 +-
.../org/apache/tajo/storage/StorageUtil.java | 3 +-
.../apache/tajo/storage/TableStatistics.java | 20 +-
.../storage/annotation/ForSplitableStore.java | 29 ++
.../tajo/storage/json/FragmentDeserializer.java | 54 ----
.../tajo/storage/json/StorageGsonHelper.java | 82 ------
.../tajo/storage/rcfile/RCFileWrapper.java | 17 +-
.../tajo/storage/trevni/TrevniAppender.java | 13 +-
.../tajo/storage/trevni/TrevniScanner.java | 5 +-
.../apache/tajo/storage/v2/CSVFileScanner.java | 9 +-
.../apache/tajo/storage/v2/FileScannerV2.java | 3 +-
.../apache/tajo/storage/v2/RCFileScanner.java | 10 +-
.../tajo/storage/v2/StorageManagerV2.java | 24 +-
.../tajo/storage/TestCompressionStorages.java | 26 +-
.../apache/tajo/storage/TestMergeScanner.java | 18 +-
.../apache/tajo/storage/TestStorageManager.java | 6 +-
.../org/apache/tajo/storage/TestStorages.java | 32 +--
.../apache/tajo/storage/index/TestBSTIndex.java | 47 ++--
.../index/TestSingleCSVFileBSTIndex.java | 20 +-
.../tajo/storage/v2/TestCSVCompression.java | 29 +-
.../apache/tajo/storage/v2/TestCSVScanner.java | 14 +-
.../apache/tajo/storage/v2/TestStorages.java | 32 +--
129 files changed, 1668 insertions(+), 2329 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7defd7b..c2a9eeb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -59,7 +59,10 @@ Release 0.2.0 - unreleased
IMPROVEMENTS
- TAJO-275: Separating QueryMaster and TaskRunner roles in worker. (Keuntae Park via jihoon)
+ TAJO-287: Refactor TableDesc, TableMeta, and Fragment. (hyunsik)
+
+ TAJO-275: Separating QueryMaster and TaskRunner roles in worker.
+ (Keuntae Park via jihoon)
TAJO-270: Boolean datum compatible to apache hive. (jinho)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
index 02650bc..c6bd7fb 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java
@@ -30,7 +30,7 @@ public interface CatalogService {
* Get a table description by name
* @param name table name
* @return a table description
- * @see org.apache.tajo.catalog.TableDescImpl
+ * @see TableDesc
* @throws Throwable
*/
TableDesc getTableDesc(String name);
@@ -51,7 +51,7 @@ public interface CatalogService {
/**
* Add a table via table description
- * @see org.apache.tajo.catalog.TableDescImpl
+ * @see TableDesc
* @throws Throwable
*/
boolean addTable(TableDesc desc);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 6e17f42..6b4848c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -19,16 +19,11 @@
package org.apache.tajo.catalog;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
-import org.apache.tajo.catalog.statistics.TableStat;
import org.apache.tajo.common.TajoDataTypes.DataType;
-import org.apache.tajo.util.FileUtil;
-import java.io.File;
-import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
@@ -58,35 +53,6 @@ public class CatalogUtil {
return sb.toString();
}
- public static String prettyPrint(TableMeta meta) {
- StringBuilder sb = new StringBuilder();
- sb.append("store type: ").append(meta.getStoreType()).append("\n");
- sb.append("schema: \n");
-
- for(int i = 0; i < meta.getSchema().getColumnNum(); i++) {
- Column col = meta.getSchema().getColumn(i);
- sb.append(col.getColumnName()).append("\t").append(col.getDataType());
- sb.append("\n");
- }
- return sb.toString();
- }
-
- public static void printTableMeta(File file) throws IOException {
- CatalogProtos.TableProto proto = (CatalogProtos.TableProto) FileUtil.
- loadProto(file, CatalogProtos.TableProto.getDefaultInstance());
- System.out.println(prettyPrint(new TableMetaImpl(proto)));
- }
-
- public static void main(String [] args) throws IOException {
- if (args.length < 2) {
- System.out.println("catalog print [filename]");
- System.exit(-1);
- }
-
- File file = new File(args[1]);
- printTableMeta(file);
- }
-
public static StoreType getStoreType(final String typeStr) {
if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) {
return StoreType.CSV;
@@ -105,32 +71,24 @@ public class CatalogUtil {
}
}
- public static TableMeta newTableMeta(Schema schema, StoreType type) {
- return new TableMetaImpl(schema, type, new Options());
- }
-
- public static TableMeta newTableMeta(Schema schema, StoreType type,
- Options options) {
- return new TableMetaImpl(schema, type, options);
+ public static TableMeta newTableMeta(StoreType type) {
+ return new TableMeta(type, new Options());
}
- public static TableMeta newTableMeta(Schema schema, StoreType type, Options options,
- TableStat stat) {
- return new TableMetaImpl(schema, type, options, stat);
+ public static TableMeta newTableMeta(StoreType type, Options options) {
+ return new TableMeta(type, options);
}
- public static TableDesc newTableDesc(String tableName, TableMeta meta,
- Path path) {
- return new TableDescImpl(tableName, meta, path);
+ public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path) {
+ return new TableDesc(tableName, schema, meta, path);
}
public static TableDesc newTableDesc(TableDescProto proto) {
- return new TableDescImpl(proto);
+ return new TableDesc(proto);
}
- public static TableDesc newTableDesc(String tableName,
- Schema schema, StoreType type, Options options, Path path) {
- return new TableDescImpl(tableName, schema, type, options, path);
+ public static TableDesc newTableDesc(String tableName, Schema schema, StoreType type, Options options, Path path) {
+ return new TableDesc(tableName, schema, type, options, path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index 87d32ca..e6cc46d 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -33,7 +33,7 @@ public class DDLBuilder {
sb.append("\n-- Path: ").append(desc.getPath());
sb.append("\n--\n");
sb.append("CREATE EXTERNAL TABLE ").append(desc.getName());
- buildSchema(sb, desc.getMeta().getSchema());
+ buildSchema(sb, desc.getSchema());
buildUsingClause(sb, desc.getMeta());
buildWithClause(sb, desc.getMeta());
buildLocationClause(sb, desc);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index 42b7e73..f59feef 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@ -18,26 +18,144 @@
package org.apache.tajo.catalog;
-import com.google.protobuf.Message;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.Expose;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.json.CatalogGsonHelper;
+import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
+import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.json.GsonObject;
-public interface TableDesc extends Cloneable, GsonObject {
- void setName(String tableId);
+public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Cloneable {
+ protected TableDescProto.Builder builder = null;
- String getName();
+ @Expose protected String tableName; // required
+ @Expose protected Schema schema;
+ @Expose protected TableMeta meta; // required
+ @Expose protected Path uri; // required
+ @Expose protected TableStats stats; // optional
- void setPath(Path path);
-
- Path getPath();
-
- void setMeta(TableMeta info);
-
- TableMeta getMeta();
+ public TableDesc() {
+ builder = TableDescProto.newBuilder();
+ }
+
+ public TableDesc(String tableName, Schema schema, TableMeta info, Path path) {
+ this();
+ // tajo deems all identifiers as lowcase characters
+ this.tableName = tableName.toLowerCase();
+ this.schema = schema;
+ this.meta = info;
+ this.uri = path;
+ }
+
+ public TableDesc(String tableName, Schema schema, StoreType type, Options options, Path path) {
+ this(tableName, schema, new TableMeta(type, options), path);
+ }
+
+ public TableDesc(TableDescProto proto) {
+ this(proto.getId(), new Schema(proto.getSchema()), new TableMeta(proto.getMeta()), new Path(proto.getPath()));
+ this.stats = new TableStats(proto.getStats());
+ }
+
+ public void setName(String tableId) {
+ // tajo deems all identifiers as lowcase characters
+ this.tableName = tableId.toLowerCase();
+ }
+
+ public String getName() {
+ return this.tableName;
+ }
+
+ public void setPath(Path uri) {
+ this.uri = uri;
+ }
+
+ public Path getPath() {
+ return this.uri;
+ }
- Schema getSchema();
-
- Message getProto();
-
- Object clone() throws CloneNotSupportedException;
+ public void setMeta(TableMeta info) {
+ this.meta = info;
+ }
+
+ public TableMeta getMeta() {
+ return this.meta;
+ }
+
+ public void setSchema(Schema schem) {
+ this.schema = schem;
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ public void setStats(TableStats stats) {
+ this.stats = stats;
+ }
+
+ public boolean hasStats() {
+ return this.stats != null;
+ }
+
+ public TableStats getStats() {
+ return this.stats;
+ }
+
+ public boolean equals(Object object) {
+ if(object instanceof TableDesc) {
+ TableDesc other = (TableDesc) object;
+
+ return this.getProto().equals(other.getProto());
+ }
+
+ return false;
+ }
+
+ public Object clone() throws CloneNotSupportedException {
+ TableDesc desc = (TableDesc) super.clone();
+ desc.builder = TableDescProto.newBuilder();
+ desc.tableName = tableName;
+ desc.schema = (Schema) schema.clone();
+ desc.meta = (TableMeta) meta.clone();
+ desc.uri = uri;
+ desc.stats = stats != null ? (TableStats) stats.clone() : null;
+
+ return desc;
+ }
+
+ public String toString() {
+ Gson gson = new GsonBuilder().setPrettyPrinting().
+ excludeFieldsWithoutExposeAnnotation().create();
+ return gson.toJson(this);
+ }
+
+ public String toJson() {
+ return CatalogGsonHelper.toJson(this, TableDesc.class);
+ }
+
+ public TableDescProto getProto() {
+ if (builder == null) {
+ builder = TableDescProto.newBuilder();
+ }
+ if (this.tableName != null) {
+ builder.setId(this.tableName);
+ }
+ if (this.schema != null) {
+ builder.setSchema(schema.getProto());
+ }
+ if (this.meta != null) {
+ builder.setMeta(meta.getProto());
+ }
+ if (this.uri != null) {
+ builder.setPath(this.uri.toString());
+ }
+ if (this.stats != null) {
+ builder.setStats(stats.getProto());
+ }
+ return builder.build();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDescImpl.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDescImpl.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDescImpl.java
deleted file mode 100644
index 5111116..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDescImpl.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.annotations.Expose;
-import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.json.CatalogGsonHelper;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
-import org.apache.tajo.common.ProtoObject;
-
-public class TableDescImpl implements TableDesc, ProtoObject<TableDescProto>, Cloneable {
- protected TableDescProto.Builder builder = null;
-
- @Expose protected String tableName; // required
- @Expose protected Path uri; // required
- @Expose protected TableMeta meta; // required
-
- public TableDescImpl() {
- builder = TableDescProto.newBuilder();
- }
-
- public TableDescImpl(String tableName, TableMeta info, Path path) {
- this();
- // tajo deems all identifiers as lowcase characters
- this.tableName = tableName.toLowerCase();
- this.meta = info;
- this.uri = path;
- }
-
- public TableDescImpl(String tableName, Schema schema, StoreType type,
- Options options, Path path) {
- this(tableName, new TableMetaImpl(schema, type, options), path);
- }
-
- public TableDescImpl(TableDescProto proto) {
- this(proto.getId(), new TableMetaImpl(proto.getMeta()), new Path(proto.getPath()));
- }
-
- public void setName(String tableId) {
- // tajo deems all identifiers as lowcase characters
- this.tableName = tableId.toLowerCase();
- }
-
- public String getName() {
- return this.tableName;
- }
-
- public void setPath(Path uri) {
- this.uri = uri;
- }
-
- public Path getPath() {
- return this.uri;
- }
-
- @Override
- public void setMeta(TableMeta info) {
- this.meta = info;
- }
-
- public TableMeta getMeta() {
- return this.meta;
- }
-
- public Schema getSchema() {
- return getMeta().getSchema();
- }
-
- public boolean equals(Object object) {
- if(object instanceof TableDescImpl) {
- TableDescImpl other = (TableDescImpl) object;
-
- return this.getProto().equals(other.getProto());
- }
-
- return false;
- }
-
- public Object clone() throws CloneNotSupportedException {
- TableDescImpl desc = (TableDescImpl) super.clone();
- desc.builder = TableDescProto.newBuilder();
- desc.tableName = tableName;
- desc.uri = uri;
- desc.meta = (TableMeta) meta.clone();
-
- return desc;
- }
-
- public String toString() {
- Gson gson = new GsonBuilder().setPrettyPrinting().
- excludeFieldsWithoutExposeAnnotation().create();
- return gson.toJson(this);
- }
-
- public String toJson() {
- return CatalogGsonHelper.toJson(this, TableDesc.class);
- }
-
- public TableDescProto getProto() {
- if (builder == null) {
- builder = TableDescProto.newBuilder();
- }
- if (this.tableName != null) {
- builder.setId(this.tableName);
- }
- if (this.uri != null) {
- builder.setPath(this.uri.toString());
- }
- if (this.meta != null) {
- builder.setMeta(meta.getProto());
- }
- return builder.build();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
index 811bcb0..50dc148 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java
@@ -18,41 +18,183 @@
package org.apache.tajo.catalog;
+import com.google.common.base.Objects;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.catalog.json.CatalogGsonHelper;
+import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.proto.CatalogProtos.TableProtoOrBuilder;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.json.GsonObject;
import java.util.Map;
-public interface TableMeta extends ProtoObject<TableProto>, Cloneable, GsonObject {
-
- static String COMPRESSION_CODEC = "compression.codec";
-
- void setStorageType(StoreType storeType);
-
- StoreType getStoreType();
-
- void setSchema(Schema schema);
-
- Schema getSchema();
-
- void putOption(String key, String val);
-
- void setStat(TableStat stat);
-
- String getOption(String key);
-
- String getOption(String key, String defaultValue);
-
- Options getOptions();
-
- Map<String,String> toMap();
-
- TableStat getStat();
-
- Object clone() throws CloneNotSupportedException;
-
- void mergeProtoToLocal();
+/**
+ * It contains all information for scanning a fragmented table
+ */
+public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObject, Cloneable {
+ public static final String COMPRESSION_CODEC = "compression.codec";
+
+ protected TableProto.Builder builder = null;
+ private TableProto proto = TableProto.getDefaultInstance();
+ private boolean viaProto = false;
+
+ @Expose protected StoreType storeType;
+ @Expose protected Options options;
+
+ private TableMeta() {
+ builder = TableProto.newBuilder();
+ }
+
+ public TableMeta(StoreType type, Options options) {
+ this();
+ this.storeType = type;
+ this.options = new Options(options);
+ }
+
+ public TableMeta(TableProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public StoreType getStoreType() {
+ TableProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.storeType != null) {
+ return storeType;
+ }
+ if (!p.hasStoreType()) {
+ return null;
+ }
+ this.storeType = p.getStoreType();
+ return this.storeType;
+ }
+
+ public void setOptions(Options options) {
+ maybeInitBuilder();
+ this.options = options;
+ }
+
+ public void putOption(String key, String val) {
+ maybeInitBuilder();
+ options.put(key, val);
+ }
+
+ public String getOption(String key) {
+ TableProtoOrBuilder p = viaProto ? proto : builder;
+ if (options != null) {
+ return this.options.get(key);
+ }
+ if (!p.hasParams()) {
+ return null;
+ }
+ this.options = new Options(p.getParams());
+ return options.get(key);
+ }
+
+ public String getOption(String key, String defaultValue) {
+ TableProtoOrBuilder p = viaProto ? proto : builder;
+ if (options != null) {
+ return this.options.get(key, defaultValue);
+ }
+ if (!p.hasParams()) {
+ return null;
+ }
+ this.options = new Options(p.getParams());
+ return options.get(key, defaultValue);
+ }
+
+ public Options getOptions() {
+ TableProtoOrBuilder p = viaProto ? proto : builder;
+ if (options != null) {
+ return this.options;
+ }
+ if (!p.hasParams()) {
+ return null;
+ }
+ this.options = new Options(p.getParams());
+ return options;
+ }
+
+ public Map<String,String> toMap() {
+ return getOptions().getAllKeyValus();
+ }
+
+ public boolean equals(Object object) {
+ if(object instanceof TableMeta) {
+ TableMeta other = (TableMeta) object;
+
+ return this.getProto().equals(other.getProto());
+ }
+
+ return false;
+ }
+
+ public int hashCode() {
+ return Objects.hashCode(getStoreType(), getOptions());
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ TableMeta meta = (TableMeta) super.clone();
+ meta.builder = TableProto.newBuilder();
+ meta.storeType = getStoreType();
+ meta.options = (Options) (toMap() != null ? options.clone() : null);
+
+ return meta;
+ }
+
+ public String toString() {
+ Gson gson = new GsonBuilder().setPrettyPrinting().
+ excludeFieldsWithoutExposeAnnotation().create();
+ return gson.toJson(this);
+ }
+
+ ////////////////////////////////////////////////////////////////////////
+ // ProtoObject
+ ////////////////////////////////////////////////////////////////////////
+ public TableProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public String toJson() {
+ mergeProtoToLocal();
+ return CatalogGsonHelper.toJson(this, TableMeta.class);
+ }
+
+ public void mergeProtoToLocal() {
+ getStoreType();
+ toMap();
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = TableProto.newBuilder(proto);
+ }
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (storeType != null) {
+ builder.setStoreType(storeType);
+ }
+ if (this.options != null) {
+ builder.setParams(options.getProto());
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if(viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMetaImpl.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMetaImpl.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMetaImpl.java
deleted file mode 100644
index 0bf1615..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMetaImpl.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog;
-
-import com.google.common.base.Objects;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.annotations.Expose;
-import org.apache.tajo.catalog.json.CatalogGsonHelper;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableProtoOrBuilder;
-import org.apache.tajo.catalog.statistics.TableStat;
-import org.apache.tajo.json.GsonObject;
-
-import java.util.Map;
-
-public class TableMetaImpl implements TableMeta, GsonObject {
- protected TableProto.Builder builder = null;
- private TableProto proto = TableProto.getDefaultInstance();
- private boolean viaProto = false;
-
- @Expose protected Schema schema;
- @Expose protected StoreType storeType;
- @Expose protected Options options;
- @Expose protected TableStat stat;
-
- private TableMetaImpl() {
- builder = TableProto.newBuilder();
- }
-
- public TableMetaImpl(Schema schema, StoreType type, Options options) {
- this();
- this.schema = schema;
- this.storeType = type;
- this.options = new Options(options);
- }
-
- public TableMetaImpl(Schema schema, StoreType type, Options options,
- TableStat stat) {
- this();
- this.schema = schema;
- this.storeType = type;
- this.options = new Options(options);
- this.stat = stat;
- }
-
- public TableMetaImpl(TableProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public void setStorageType(StoreType storeType) {
- maybeInitBuilder();
- this.storeType = storeType;
- }
-
- public StoreType getStoreType() {
- TableProtoOrBuilder p = viaProto ? proto : builder;
- if (this.storeType != null) {
- return storeType;
- }
- if (!p.hasStoreType()) {
- return null;
- }
- this.storeType = p.getStoreType();
- return this.storeType;
- }
-
- public void setSchema(Schema schema) {
- maybeInitBuilder();
- this.schema = schema;
- }
-
- public void setOptions(Options options) {
- maybeInitBuilder();
- this.options = options;
- }
-
- @Override
- public void putOption(String key, String val) {
- maybeInitBuilder();
- options.put(key, val);
- }
-
- public Schema getSchema() {
- TableProtoOrBuilder p = viaProto ? proto : builder;
- if (schema != null) {
- return this.schema;
- }
- if (!p.hasSchema()) {
- return null;
- }
- this.schema = new Schema(p.getSchema());
- return this.schema;
- }
-
- @Override
- public String getOption(String key) {
- TableProtoOrBuilder p = viaProto ? proto : builder;
- if (options != null) {
- return this.options.get(key);
- }
- if (!p.hasParams()) {
- return null;
- }
- this.options = new Options(p.getParams());
- return options.get(key);
- }
-
- @Override
- public String getOption(String key, String defaultValue) {
- TableProtoOrBuilder p = viaProto ? proto : builder;
- if (options != null) {
- return this.options.get(key, defaultValue);
- }
- if (!p.hasParams()) {
- return null;
- }
- this.options = new Options(p.getParams());
- return options.get(key, defaultValue);
- }
-
- @Override
- public Options getOptions() {
- TableProtoOrBuilder p = viaProto ? proto : builder;
- if (options != null) {
- return this.options;
- }
- if (!p.hasParams()) {
- return null;
- }
- this.options = new Options(p.getParams());
- return options;
- }
-
- @Override
- public Map<String,String> toMap() {
- return getOptions().getAllKeyValus();
- }
-
- public boolean equals(Object object) {
- if(object instanceof TableMetaImpl) {
- TableMetaImpl other = (TableMetaImpl) object;
-
- return this.getProto().equals(other.getProto());
- }
-
- return false;
- }
-
- public int hashCode() {
- return Objects.hashCode(getSchema(), getStoreType());
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- TableMetaImpl meta = (TableMetaImpl) super.clone();
- meta.builder = TableProto.newBuilder();
- meta.schema = (Schema) getSchema().clone();
- meta.storeType = getStoreType();
- meta.stat = (TableStat) (getStat() != null ? stat.clone() : null);
- meta.options = (Options) (toMap() != null ? options.clone() : null);
-
- return meta;
- }
-
- public String toString() {
- Gson gson = new GsonBuilder().setPrettyPrinting().
- excludeFieldsWithoutExposeAnnotation().create();
- return gson.toJson(this);
- }
-
- ////////////////////////////////////////////////////////////////////////
- // ProtoObject
- ////////////////////////////////////////////////////////////////////////
- @Override
- public TableProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public String toJson() {
- mergeProtoToLocal();
- return CatalogGsonHelper.toJson(this, TableMeta.class);
- }
-
- public void mergeProtoToLocal() {
- getSchema();
- getStoreType();
- toMap();
- getStat();
- }
-
- @Override
- public void setStat(TableStat stat) {
- maybeInitBuilder();
- this.stat = stat;
- }
-
- @Override
- public TableStat getStat() {
- TableProtoOrBuilder p = viaProto ? proto : builder;
- if (this.stat != null) {
- return this.stat;
- }
- if (!p.hasStat()) {
- return null;
- }
- this.stat = new TableStat(p.getStat());
- return this.stat;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = TableProto.newBuilder(proto);
- }
- viaProto = true;
- }
-
- private void mergeLocalToBuilder() {
- if (schema != null) {
- builder.setSchema(schema.getProto());
- }
- if (storeType != null) {
- builder.setStoreType(storeType);
- }
- if (this.options != null) {
- builder.setParams(options.getProto());
- }
- if (this.stat != null) {
- builder.setStat(stat.getProto());
- }
- }
-
- private void mergeLocalToProto() {
- if(viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
index 6325bd2..76aec33 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
@@ -31,7 +31,7 @@ import java.io.IOException;
public class TableUtil {
public static TableMeta getTableMeta(Configuration conf, Path tablePath)
throws IOException {
- TableMetaImpl meta = null;
+ TableMeta meta = null;
FileSystem fs = tablePath.getFileSystem(conf);
@@ -44,7 +44,7 @@ public class TableUtil {
TableProto tableProto = (TableProto) FileUtil.loadProto(tableMetaIn,
TableProto.getDefaultInstance());
- meta = new TableMetaImpl(tableProto);
+ meta = new TableMeta(tableProto);
return meta;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
index 9f031f4..94bfdbe 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java
@@ -21,7 +21,6 @@ package org.apache.tajo.catalog.json;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.function.Function;
import org.apache.tajo.common.TajoDataTypes.DataType;
@@ -43,7 +42,6 @@ public class CatalogGsonHelper {
Map<Type, GsonSerDerAdapter> adapters = TUtil.newHashMap();
adapters.put(Class.class, new ClassNameSerializer());
adapters.put(Path.class, new PathSerializer());
- adapters.put(TableDesc.class, new TableDescAdapter());
adapters.put(TableMeta.class, new TableMetaAdapter());
adapters.put(Function.class, new FunctionAdapter());
adapters.put(Datum.class, new DatumAdapter());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableDescAdapter.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableDescAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableDescAdapter.java
deleted file mode 100644
index f942048..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableDescAdapter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog.json;
-
-import com.google.common.base.Preconditions;
-import com.google.gson.*;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableDescImpl;
-import org.apache.tajo.json.GsonSerDerAdapter;
-
-import java.lang.reflect.Type;
-
-public class TableDescAdapter implements GsonSerDerAdapter<TableDesc> {
-
- @Override
- public TableDesc deserialize(JsonElement json, Type type,
- JsonDeserializationContext ctx) throws JsonParseException {
- JsonObject jsonObject = json.getAsJsonObject();
- String typeName = jsonObject.get("type").getAsJsonPrimitive().getAsString();
- Preconditions.checkState(typeName.equals("TableDesc"));
- return ctx.deserialize(jsonObject.get("body"), TableDescImpl.class);
- }
-
- @Override
- public JsonElement serialize(TableDesc src, Type typeOfSrc,
- JsonSerializationContext context) {
- JsonObject jsonObj = new JsonObject();
- jsonObj.addProperty("type", "TableDesc");
- JsonElement jsonElem = context.serialize(src, TableDescImpl.class);
- jsonObj.add("body", jsonElem);
- return jsonObj;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
index f01a6c1..a3337c8 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java
@@ -18,10 +18,10 @@
package org.apache.tajo.catalog.json;
-import com.google.common.base.Preconditions;
import com.google.gson.*;
+import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.TableMetaImpl;
+import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.json.GsonSerDerAdapter;
import java.lang.reflect.Type;
@@ -32,19 +32,22 @@ public class TableMetaAdapter implements GsonSerDerAdapter<TableMeta> {
public TableMeta deserialize(JsonElement json, Type typeOfT,
JsonDeserializationContext context) throws JsonParseException {
JsonObject jsonObject = json.getAsJsonObject();
- String typeName = jsonObject.get("type").getAsJsonPrimitive().getAsString();
- Preconditions.checkState(typeName.equals("TableMeta"));
- return context.deserialize(jsonObject.get("body"), TableMetaImpl.class);
+
+ CatalogProtos.StoreType type = CatalogProtos.StoreType.valueOf(jsonObject.get("store").getAsString());
+
+ Options options = null;
+ if (jsonObject.has("options")) {
+ options = context.deserialize(jsonObject.get("options"), Options.class);
+ }
+ return new TableMeta(type, options);
}
@Override
public JsonElement serialize(TableMeta src, Type typeOfSrc,
JsonSerializationContext context) {
JsonObject jsonObj = new JsonObject();
- jsonObj.addProperty("type", "TableMeta");
- src.mergeProtoToLocal();
- JsonElement jsonElem = context.serialize(src, TableMetaImpl.class);
- jsonObj.add("body", jsonElem);
+ jsonObj.addProperty("store", src.getStoreType().name());
+ jsonObj.add("options", context.serialize(src.getOptions(), Options.class));
return jsonObj;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStat.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStat.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStat.java
deleted file mode 100644
index f7793c3..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStat.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- *
- */
-package org.apache.tajo.catalog.statistics;
-
-import com.google.common.base.Objects;
-import com.google.gson.annotations.Expose;
-import com.google.protobuf.ByteString;
-import org.apache.tajo.json.GsonObject;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.json.CatalogGsonHelper;
-import org.apache.tajo.catalog.proto.CatalogProtos.ColumnStatProto;
-import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.util.TUtil;
-
-public class ColumnStat implements ProtoObject<ColumnStatProto>, Cloneable, GsonObject {
- private ColumnStatProto.Builder builder = ColumnStatProto.newBuilder();
-
- @Expose private Column column = null; // required
- @Expose private Long numDistVals = null; // optional
- @Expose private Long numNulls = null; // optional
- @Expose private Datum minValue = null; // optional
- @Expose private Datum maxValue = null; // optional
-
- public ColumnStat(Column column) {
- this.column = column;
- numDistVals = 0l;
- numNulls = 0l;
- }
-
- public ColumnStat(ColumnStatProto proto) {
- this.column = new Column(proto.getColumn());
-
- if (proto.hasNumDistVal()) {
- this.numDistVals = proto.getNumDistVal();
- }
- if (proto.hasNumNulls()) {
- this.numNulls = proto.getNumNulls();
- }
- if (proto.hasMinValue()) {
- this.minValue = TupleUtil.createFromBytes(getColumn().getDataType(), proto.getMinValue().toByteArray());
- }
- if (proto.hasMaxValue()) {
- this.maxValue = TupleUtil.createFromBytes(getColumn().getDataType(), proto.getMaxValue().toByteArray());
- }
- }
-
- public Column getColumn() {
- return this.column;
- }
-
- public Long getNumDistValues() {
- return this.numDistVals;
- }
-
- public void setNumDistVals(long numDistVals) {
- this.numDistVals = numDistVals;
- }
-
- public boolean minIsNotSet() {
- return minValue == null;
- }
-
- public Datum getMinValue() {
- return this.minValue;
- }
-
- public void setMinValue(Datum minValue) {
- this.minValue = minValue;
- }
-
- public boolean maxIsNotSet() {
- return maxValue == null;
- }
-
- public Datum getMaxValue() {
- return this.maxValue;
- }
-
- public void setMaxValue(Datum maxValue) {
- this.maxValue = maxValue;
- }
-
- public Long getNumNulls() {
- return this.numNulls;
- }
-
- public void setNumNulls(long numNulls) {
- this.numNulls = numNulls;
- }
-
- public boolean equals(Object obj) {
- if (obj instanceof ColumnStat) {
- ColumnStat other = (ColumnStat) obj;
- return getColumn().equals(other.getColumn())
- && getNumDistValues().equals(other.getNumDistValues())
- && getNumNulls().equals(other.getNumNulls())
- && TUtil.checkEquals(getMinValue(), other.getMinValue())
- && TUtil.checkEquals(getMaxValue(), other.getMaxValue());
- } else {
- return false;
- }
- }
-
- public int hashCode() {
- return Objects.hashCode(getNumDistValues(), getNumNulls());
- }
-
- public Object clone() throws CloneNotSupportedException {
- ColumnStat stat = (ColumnStat) super.clone();
- stat.builder = ColumnStatProto.newBuilder();
- stat.column = (Column) this.column.clone();
- stat.numDistVals = numDistVals;
- stat.numNulls = numNulls;
- stat.minValue = minValue;
- stat.maxValue = maxValue;
-
- return stat;
- }
-
- public String toString() {
- return CatalogGsonHelper.getPrettyInstance().toJson(this, ColumnStat.class);
- }
-
- @Override
- public String toJson() {
- return CatalogGsonHelper.toJson(this, ColumnStat.class);
- }
-
-
- @Override
- public ColumnStatProto getProto() {
- if (builder == null) {
- builder = ColumnStatProto.newBuilder();
- }
- if (this.column != null) {
- builder.setColumn(this.column.getProto());
- }
- if (this.numDistVals != null) {
- builder.setNumDistVal(this.numDistVals);
- }
- if (this.numNulls != null) {
- builder.setNumNulls(this.numNulls);
- }
- if (this.minValue != null) {
- builder.setMinValue(ByteString.copyFrom(this.minValue.asByteArray()));
- }
- if (this.maxValue != null) {
- builder.setMaxValue(ByteString.copyFrom(this.maxValue.asByteArray()));
- }
-
- return builder.build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
new file mode 100644
index 0000000..017c688
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.tajo.catalog.statistics;
+
+import com.google.common.base.Objects;
+import com.google.gson.annotations.Expose;
+import com.google.protobuf.ByteString;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.json.GsonObject;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.json.CatalogGsonHelper;
+import org.apache.tajo.common.ProtoObject;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.util.TUtil;
+
+public class ColumnStats implements ProtoObject<CatalogProtos.ColumnStatsProto>, Cloneable, GsonObject {
+ private CatalogProtos.ColumnStatsProto.Builder builder = CatalogProtos.ColumnStatsProto.newBuilder();
+
+ @Expose private Column column = null; // required
+ @Expose private Long numDistVals = null; // optional
+ @Expose private Long numNulls = null; // optional
+ @Expose private Datum minValue = null; // optional
+ @Expose private Datum maxValue = null; // optional
+
+ public ColumnStats(Column column) {
+ this.column = column;
+ numDistVals = 0l;
+ numNulls = 0l;
+ }
+
+ public ColumnStats(CatalogProtos.ColumnStatsProto proto) {
+ this.column = new Column(proto.getColumn());
+
+ if (proto.hasNumDistVal()) {
+ this.numDistVals = proto.getNumDistVal();
+ }
+ if (proto.hasNumNulls()) {
+ this.numNulls = proto.getNumNulls();
+ }
+ if (proto.hasMinValue()) {
+ this.minValue = TupleUtil.createFromBytes(getColumn().getDataType(), proto.getMinValue().toByteArray());
+ }
+ if (proto.hasMaxValue()) {
+ this.maxValue = TupleUtil.createFromBytes(getColumn().getDataType(), proto.getMaxValue().toByteArray());
+ }
+ }
+
+ public Column getColumn() {
+ return this.column;
+ }
+
+ public Long getNumDistValues() {
+ return this.numDistVals;
+ }
+
+ public void setNumDistVals(long numDistVals) {
+ this.numDistVals = numDistVals;
+ }
+
+ public boolean minIsNotSet() {
+ return minValue == null;
+ }
+
+ public Datum getMinValue() {
+ return this.minValue;
+ }
+
+ public void setMinValue(Datum minValue) {
+ this.minValue = minValue;
+ }
+
+ public boolean maxIsNotSet() {
+ return maxValue == null;
+ }
+
+ public Datum getMaxValue() {
+ return this.maxValue;
+ }
+
+ public void setMaxValue(Datum maxValue) {
+ this.maxValue = maxValue;
+ }
+
+ public Long getNumNulls() {
+ return this.numNulls;
+ }
+
+ public void setNumNulls(long numNulls) {
+ this.numNulls = numNulls;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj instanceof ColumnStats) {
+ ColumnStats other = (ColumnStats) obj;
+ return getColumn().equals(other.getColumn())
+ && getNumDistValues().equals(other.getNumDistValues())
+ && getNumNulls().equals(other.getNumNulls())
+ && TUtil.checkEquals(getMinValue(), other.getMinValue())
+ && TUtil.checkEquals(getMaxValue(), other.getMaxValue());
+ } else {
+ return false;
+ }
+ }
+
+ public int hashCode() {
+ return Objects.hashCode(getNumDistValues(), getNumNulls());
+ }
+
+ public Object clone() throws CloneNotSupportedException {
+ ColumnStats stat = (ColumnStats) super.clone();
+ stat.builder = CatalogProtos.ColumnStatsProto.newBuilder();
+ stat.column = (Column) this.column.clone();
+ stat.numDistVals = numDistVals;
+ stat.numNulls = numNulls;
+ stat.minValue = minValue;
+ stat.maxValue = maxValue;
+
+ return stat;
+ }
+
+ public String toString() {
+ return CatalogGsonHelper.getPrettyInstance().toJson(this, ColumnStats.class);
+ }
+
+ @Override
+ public String toJson() {
+ return CatalogGsonHelper.toJson(this, ColumnStats.class);
+ }
+
+
+ @Override
+ public CatalogProtos.ColumnStatsProto getProto() {
+ if (builder == null) {
+ builder = CatalogProtos.ColumnStatsProto.newBuilder();
+ }
+ if (this.column != null) {
+ builder.setColumn(this.column.getProto());
+ }
+ if (this.numDistVals != null) {
+ builder.setNumDistVal(this.numDistVals);
+ }
+ if (this.numNulls != null) {
+ builder.setNumNulls(this.numNulls);
+ }
+ if (this.minValue != null) {
+ builder.setMinValue(ByteString.copyFrom(this.minValue.asByteArray()));
+ }
+ if (this.maxValue != null) {
+ builder.setMaxValue(ByteString.copyFrom(this.maxValue.asByteArray()));
+ }
+
+ return builder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
index 6f2fdf6..014b67c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
@@ -44,32 +44,32 @@ public class StatisticsUtil {
return aggregated;
}
- public static TableStat aggregateTableStat(List<TableStat> tableStats) {
- if(tableStats == null || tableStats.size() == 0 || tableStats.get(0) == null)
+ public static TableStats aggregateTableStat(List<TableStats> tableStatses) {
+ if(tableStatses == null || tableStatses.size() == 0 || tableStatses.get(0) == null)
return null;
- TableStat aggregated = new TableStat();
+ TableStats aggregated = new TableStats();
- ColumnStat [] css = null;
- if (tableStats.size() > 0) {
- for (TableStat ts : tableStats) {
+ ColumnStats[] css = null;
+ if (tableStatses.size() > 0) {
+ for (TableStats ts : tableStatses) {
// A TableStats cannot contain any ColumnStat if there is no output.
// So, we should consider such a condition.
if (ts.getColumnStats().size() > 0) {
- css = new ColumnStat[ts.getColumnStats().size()];
+ css = new ColumnStats[ts.getColumnStats().size()];
for (int i = 0; i < css.length; i++) {
- css[i] = new ColumnStat(ts.getColumnStats().get(i).getColumn());
+ css[i] = new ColumnStats(ts.getColumnStats().get(i).getColumn());
}
break;
}
}
}
- for (TableStat ts : tableStats) {
+ for (TableStats ts : tableStatses) {
// if there is empty stats
if (ts.getColumnStats().size() > 0) {
// aggregate column stats for each table
for (int i = 0; i < ts.getColumnStats().size(); i++) {
- ColumnStat cs = ts.getColumnStats().get(i);
+ ColumnStats cs = ts.getColumnStats().get(i);
if (cs == null) {
LOG.warn("ERROR: One of column stats is NULL (expected column: " + css[i].getColumn() + ")");
continue;
@@ -102,25 +102,25 @@ public class StatisticsUtil {
return aggregated;
}
- public static TableStat computeStatFromUnionBlock(Collection<TableStat> stats) {
- TableStat stat = new TableStat();
- TableStat childStat;
+ public static TableStats computeStatFromUnionBlock(Collection<TableStats> stats) {
+ TableStats stat = new TableStats();
+ TableStats childStat;
long avgRows = 0, numBytes = 0, numRows = 0;
int numBlocks = 0, numPartitions = 0;
- List<ColumnStat> columnStats = Lists.newArrayList();
+ List<ColumnStats> columnStatses = Lists.newArrayList();
- Iterator<TableStat> it = stats.iterator();
+ Iterator<TableStats> it = stats.iterator();
while (it.hasNext()) {
childStat = it.next();
avgRows += childStat.getAvgRows();
- columnStats.addAll(childStat.getColumnStats());
+ columnStatses.addAll(childStat.getColumnStats());
numBlocks += childStat.getNumBlocks();
numBytes += childStat.getNumBytes();
numPartitions += childStat.getNumPartitions();
numRows += childStat.getNumRows();
}
- stat.setColumnStats(columnStats);
+ stat.setColumnStats(columnStatses);
stat.setNumBlocks(numBlocks);
stat.setNumBytes(numBytes);
stat.setNumPartitions(numPartitions);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStat.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStat.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStat.java
deleted file mode 100644
index ac6a178..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStat.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- *
- */
-package org.apache.tajo.catalog.statistics;
-
-import com.google.common.base.Objects;
-import com.google.gson.Gson;
-import com.google.gson.annotations.Expose;
-import org.apache.tajo.common.TajoDataTypes;
-import org.apache.tajo.json.GsonObject;
-import org.apache.tajo.catalog.json.CatalogGsonHelper;
-import org.apache.tajo.catalog.proto.CatalogProtos.ColumnStatProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableStatProto;
-import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.util.TUtil;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class TableStat implements ProtoObject<TableStatProto>, Cloneable, GsonObject {
- private TableStatProto.Builder builder = TableStatProto.newBuilder();
-
- @Expose private Long numRows = null; // required
- @Expose private Long numBytes = null; // required
- @Expose private Integer numFiles = null; // required
- @Expose private Integer numBlocks = null; // optional
- @Expose private Integer numPartitions = null; // optional
- @Expose private Long avgRows = null; // optional
- @Expose private List<ColumnStat> columnStats = null; // repeated
-
- public TableStat() {
- numRows = 0l;
- numBytes = 0l;
- numFiles = 0;
- numBlocks = 0;
- numPartitions = 0;
- avgRows = 0l;
- columnStats = TUtil.newList();
- }
-
- public TableStat(TableStatProto proto) {
- this.numRows = proto.getNumRows();
- this.numBytes = proto.getNumBytes();
- //this.numFiles =
-
- if (proto.hasNumBlocks()) {
- this.numBlocks = proto.getNumBlocks();
- }
- if (proto.hasNumPartitions()) {
- this.numPartitions = proto.getNumPartitions();
- }
- if (proto.hasAvgRows()) {
- this.avgRows = proto.getAvgRows();
- }
-
- this.columnStats = TUtil.newList();
- for (ColumnStatProto colProto : proto.getColStatList()) {
- if (colProto.getColumn().getDataType().getType() == TajoDataTypes.Type.PROTOBUF) {
- continue;
- }
- columnStats.add(new ColumnStat(colProto));
- }
- }
-
- public Long getNumRows() {
- return this.numRows;
- }
-
- public void setNumRows(long numRows) {
- this.numRows = numRows;
- }
-
- public Integer getNumBlocks() {
- return this.numBlocks;
- }
-
- public void setNumBytes(long numBytes) {
- this.numBytes = numBytes;
- }
-
- public Long getNumBytes() {
- return this.numBytes;
- }
-
- public void setNumBlocks(int numBlocks) {
- this.numBlocks = numBlocks;
- }
-
- public Integer getNumPartitions() {
- return this.numPartitions;
- }
-
- public void setNumPartitions(int numPartitions) {
- this.numPartitions = numPartitions;
- }
-
- public Long getAvgRows() {
- return this.avgRows;
- }
-
- public void setAvgRows(long avgRows) {
- this.avgRows = avgRows;
- }
-
- public List<ColumnStat> getColumnStats() {
- return this.columnStats;
- }
-
- public void setColumnStats(List<ColumnStat> columnStats) {
- this.columnStats = new ArrayList<ColumnStat>(columnStats);
- }
-
- public void addColumnStat(ColumnStat columnStat) {
- this.columnStats.add(columnStat);
- }
-
- public boolean equals(Object obj) {
- if (obj instanceof TableStat) {
- TableStat other = (TableStat) obj;
-
- return this.numRows.equals(other.numRows)
- && this.numBytes.equals(other.numBytes)
- && TUtil.checkEquals(this.numBlocks, other.numBlocks)
- && TUtil.checkEquals(this.numPartitions, other.numPartitions)
- && TUtil.checkEquals(this.avgRows, other.avgRows)
- && TUtil.checkEquals(this.columnStats, other.columnStats);
- } else {
- return false;
- }
- }
-
- public int hashCode() {
- return Objects.hashCode(numRows, numBytes,
- numBlocks, numPartitions, columnStats);
- }
-
- public Object clone() throws CloneNotSupportedException {
- TableStat stat = (TableStat) super.clone();
- stat.builder = TableStatProto.newBuilder();
- stat.numRows = numRows;
- stat.numBytes = numBytes;
- stat.numBlocks = numBlocks;
- stat.numPartitions = numPartitions;
- stat.columnStats = new ArrayList<ColumnStat>(this.columnStats);
-
- return stat;
- }
-
- public String toString() {
- Gson gson = CatalogGsonHelper.getPrettyInstance();
- return gson.toJson(this);
- }
-
- @Override
- public String toJson() {
- return CatalogGsonHelper.toJson(this, TableStat.class);
- }
-
- @Override
- public TableStatProto getProto() {
- if (builder == null) {
- builder = TableStatProto.newBuilder();
- } else {
- builder.clear();
- }
-
- builder.setNumRows(this.numRows);
- builder.setNumBytes(this.numBytes);
-
- if (this.numBlocks != null) {
- builder.setNumBlocks(this.numBlocks);
- }
- if (this.numPartitions != null) {
- builder.setNumPartitions(this.numPartitions);
- }
- if (this.avgRows != null) {
- builder.setAvgRows(this.avgRows);
- }
- if (this.columnStats != null) {
- for (ColumnStat colStat : columnStats) {
- builder.addColStat(colStat.getProto());
- }
- }
- return builder.build();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
new file mode 100644
index 0000000..0f7bc29
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.tajo.catalog.statistics;
+
+import com.google.common.base.Objects;
+import com.google.gson.Gson;
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.json.GsonObject;
+import org.apache.tajo.catalog.json.CatalogGsonHelper;
+import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
+import org.apache.tajo.common.ProtoObject;
+import org.apache.tajo.util.TUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TableStats implements ProtoObject<TableStatsProto>, Cloneable, GsonObject {
+ private TableStatsProto.Builder builder = TableStatsProto.newBuilder();
+
+ @Expose private Long numRows = null; // required
+ @Expose private Long numBytes = null; // required
+ @Expose private Integer numBlocks = null; // optional
+ @Expose private Integer numPartitions = null; // optional
+ @Expose private Long avgRows = null; // optional
+ @Expose private List<ColumnStats> columnStatses = null; // repeated
+
+ public TableStats() {
+ numRows = 0l;
+ numBytes = 0l;
+ numBlocks = 0;
+ numPartitions = 0;
+ avgRows = 0l;
+ columnStatses = TUtil.newList();
+ }
+
+ public TableStats(CatalogProtos.TableStatsProto proto) {
+ this.numRows = proto.getNumRows();
+ this.numBytes = proto.getNumBytes();
+
+ if (proto.hasNumBlocks()) {
+ this.numBlocks = proto.getNumBlocks();
+ } else {
+ this.numBlocks = 0;
+ }
+ if (proto.hasNumPartitions()) {
+ this.numPartitions = proto.getNumPartitions();
+ } else {
+ this.numPartitions = 0;
+ }
+ if (proto.hasAvgRows()) {
+ this.avgRows = proto.getAvgRows();
+ } else {
+ this.avgRows = 0l;
+ }
+
+ this.columnStatses = TUtil.newList();
+ for (CatalogProtos.ColumnStatsProto colProto : proto.getColStatList()) {
+ if (colProto.getColumn().getDataType().getType() == TajoDataTypes.Type.PROTOBUF) {
+ continue;
+ }
+ columnStatses.add(new ColumnStats(colProto));
+ }
+ }
+
+ public Long getNumRows() {
+ return this.numRows;
+ }
+
+ public void setNumRows(long numRows) {
+ this.numRows = numRows;
+ }
+
+ public Integer getNumBlocks() {
+ return this.numBlocks;
+ }
+
+ public void setNumBytes(long numBytes) {
+ this.numBytes = numBytes;
+ }
+
+ public Long getNumBytes() {
+ return this.numBytes;
+ }
+
+ public void setNumBlocks(int numBlocks) {
+ this.numBlocks = numBlocks;
+ }
+
+ public Integer getNumPartitions() {
+ return this.numPartitions;
+ }
+
+ public void setNumPartitions(int numPartitions) {
+ this.numPartitions = numPartitions;
+ }
+
+ public Long getAvgRows() {
+ return this.avgRows;
+ }
+
+ public void setAvgRows(long avgRows) {
+ this.avgRows = avgRows;
+ }
+
+ public List<ColumnStats> getColumnStats() {
+ return this.columnStatses;
+ }
+
+ public void setColumnStats(List<ColumnStats> columnStatses) {
+ this.columnStatses = new ArrayList<ColumnStats>(columnStatses);
+ }
+
+ public void addColumnStat(ColumnStats columnStats) {
+ this.columnStatses.add(columnStats);
+ }
+
+ public boolean equals(Object obj) {
+ if (obj instanceof TableStats) {
+ TableStats other = (TableStats) obj;
+
+ boolean eq = this.numRows.equals(other.numRows);
+ eq = eq && this.numBytes.equals(other.numBytes);
+ eq = eq && TUtil.checkEquals(this.numBlocks, other.numBlocks);
+ eq = eq && TUtil.checkEquals(this.numPartitions, other.numPartitions);
+ eq = eq && TUtil.checkEquals(this.avgRows, other.avgRows);
+ eq = eq && TUtil.checkEquals(this.columnStatses, other.columnStatses);
+ return eq;
+ } else {
+ return false;
+ }
+ }
+
+ public int hashCode() {
+ return Objects.hashCode(numRows, numBytes,
+ numBlocks, numPartitions, columnStatses);
+ }
+
+ public Object clone() throws CloneNotSupportedException {
+ TableStats stat = (TableStats) super.clone();
+ stat.builder = CatalogProtos.TableStatsProto.newBuilder();
+ stat.numRows = numRows != null ? numRows.longValue() : null;
+ stat.numBytes = numBytes != null ? numBytes.longValue() : null;
+ stat.numBlocks = numBlocks != null ? numBlocks.intValue() : null;
+ stat.numPartitions = numPartitions != null ? numPartitions.intValue() : null;
+ stat.columnStatses = new ArrayList<ColumnStats>(this.columnStatses);
+
+ return stat;
+ }
+
+ public String toString() {
+ Gson gson = CatalogGsonHelper.getPrettyInstance();
+ return gson.toJson(this);
+ }
+
+ @Override
+ public String toJson() {
+ return CatalogGsonHelper.toJson(this, TableStats.class);
+ }
+
+ @Override
+ public TableStatsProto getProto() {
+ if (builder == null) {
+ builder = CatalogProtos.TableStatsProto.newBuilder();
+ } else {
+ builder.clear();
+ }
+
+ builder.setNumRows(this.numRows);
+ builder.setNumBytes(this.numBytes);
+
+ if (this.numBlocks != null) {
+ builder.setNumBlocks(this.numBlocks);
+ }
+ if (this.numPartitions != null) {
+ builder.setNumPartitions(this.numPartitions);
+ }
+ if (this.avgRows != null) {
+ builder.setAvgRows(this.avgRows);
+ }
+ if (this.columnStatses != null) {
+ for (ColumnStats colStat : columnStatses) {
+ builder.addColStat(colStat.getProto());
+ }
+ }
+ return builder.build();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 1245313..3de5409 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -86,24 +86,23 @@ message FragmentProto {
required string path = 2;
required int64 startOffset = 3;
required int64 length = 4;
- required TableProto meta = 5;
- optional TableStatProto stat = 6;
- optional bool distCached = 7 [default = false];
- repeated string hosts = 8;
- repeated int32 diskIds = 9;
+ optional TableStatsProto stats = 5;
+ optional bool distCached = 6 [default = false];
+ repeated string hosts = 7;
+ repeated int32 diskIds = 8;
}
message TableProto {
- required SchemaProto schema = 1;
- required StoreType storeType = 2;
- required KeyValueSetProto params = 3;
- optional TableStatProto stat = 4;
+ required StoreType storeType = 1;
+ optional KeyValueSetProto params = 2;
}
message TableDescProto {
required string id = 1;
required string path = 2;
required TableProto meta = 3;
+ required SchemaProto schema = 4;
+ optional TableStatsProto stats = 5;
}
enum FunctionType {
@@ -169,16 +168,16 @@ message ContainFunctionRequest {
repeated DataType parameterTypes = 3;
}
-message TableStatProto {
+message TableStatsProto {
required int64 numRows = 1;
required int64 numBytes = 2;
optional int32 numBlocks = 4;
optional int32 numPartitions = 5;
optional int64 avgRows = 6;
- repeated ColumnStatProto colStat = 7;
+ repeated ColumnStatsProto colStat = 7;
}
-message ColumnStatProto {
+message ColumnStatsProto {
required ColumnProto column = 1;
optional int64 numDistVal = 2;
optional int64 numNulls = 3;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
index 64c967e..fec8e80 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
@@ -21,6 +21,8 @@ package org.apache.tajo.catalog;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.statistics.ColumnStats;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.util.CommonTestingUtil;
import org.junit.Before;
import org.junit.Test;
@@ -33,17 +35,35 @@ import static org.junit.Assert.assertEquals;
public class TestTableDesc {
TableMeta info;
+ Schema schema;
TableDesc desc;
Path path;
+ TableStats stats;
@Before
public void setup() throws IOException {
- Schema schema = new Schema();
+ schema = new Schema();
schema.addColumn("name", Type.BLOB);
schema.addColumn("addr", Type.TEXT);
- info = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ info = CatalogUtil.newTableMeta(StoreType.CSV);
path = new Path(CommonTestingUtil.getTestDir(), "table1");
- desc = new TableDescImpl("table1", info, path);
+ desc = new TableDesc("table1", schema, info, path);
+ stats = new TableStats();
+ stats.setNumRows(957685);
+ stats.setNumBytes(1023234);
+ stats.setNumBlocks(3123);
+ stats.setNumPartitions(5);
+ stats.setAvgRows(80000);
+
+ int numCols = 2;
+ ColumnStats[] cols = new ColumnStats[numCols];
+ for (int i = 0; i < numCols; i++) {
+ cols[i] = new ColumnStats(schema.getColumn(i));
+ cols[i].setNumDistVals(1024 * i);
+ cols[i].setNumNulls(100 * i);
+ stats.addColumnStat(cols[i]);
+ }
+ desc.setStats(stats);
}
@Test
@@ -51,12 +71,12 @@ public class TestTableDesc {
Schema schema = new Schema();
schema.addColumn("name", Type.BLOB);
schema.addColumn("addr", Type.TEXT);
- TableMeta info = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta info = CatalogUtil.newTableMeta(StoreType.CSV);
testClone(info);
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
- TableDesc desc = new TableDescImpl("table1", info, path);
+ TableDesc desc = new TableDesc("table1", schema, info, path);
assertEquals("table1", desc.getName());
assertEquals(path, desc.getPath());
@@ -67,17 +87,19 @@ public class TestTableDesc {
@Test
public void testGetProto() throws CloneNotSupportedException, IOException {
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
- TableDesc desc = new TableDescImpl("table1", info, path);
- CatalogProtos.TableDescProto proto = (CatalogProtos.TableDescProto) desc.getProto();
+ TableDesc desc = new TableDesc("table1", schema, info, path);
+ desc.setStats(stats);
+ CatalogProtos.TableDescProto proto = desc.getProto();
- TableDesc fromProto = new TableDescImpl(proto);
+ TableDesc fromProto = new TableDesc(proto);
assertEquals("equality check the object deserialized from json", desc, fromProto);
}
@Test
public void testToJson() throws CloneNotSupportedException, IOException {
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
- TableDesc desc = new TableDescImpl("table1", info, path);
+ TableDesc desc = new TableDesc("table1", schema, info, path);
+ desc.setStats(stats);
String json = desc.toJson();
TableDesc fromJson = CatalogGsonHelper.fromJson(json, TableDesc.class);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableInfo.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableInfo.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableInfo.java
deleted file mode 100644
index 0be3fc5..0000000
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableInfo.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.common.TajoDataTypes.Type;
-
-import static org.junit.Assert.*;
-
-public class TestTableInfo {
- TableMeta meta = null;
- Schema schema = null;
-
- @Before
- public void setUp() {
- schema = new Schema();
- schema.addColumn("name", Type.BLOB);
- schema.addColumn("addr", Type.TEXT);
- meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- }
-
- @Test
- public void testTableMetaTableProto() {
- Schema schema1 = new Schema();
- schema1.addColumn("name", Type.BLOB);
- schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
-
- TableMeta meta2 = new TableMetaImpl(meta1.getProto());
- assertEquals(meta1, meta2);
- }
-
- @Test
- public final void testClone() throws CloneNotSupportedException {
- Schema schema1 = new Schema();
- schema1.addColumn("name", Type.BLOB);
- schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
-
- TableMetaImpl info = (TableMetaImpl) meta1;
-
- TableMetaImpl info2 = (TableMetaImpl) info.clone();
- assertEquals(info.getSchema(), info2.getSchema());
- assertEquals(info.getStoreType(), info2.getStoreType());
- assertEquals(info, info2);
- }
-
- @Test
- public void testSchema() throws CloneNotSupportedException {
- Schema schema1 = new Schema();
- schema1.addColumn("name", Type.BLOB);
- schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
-
- TableMeta meta2 = (TableMeta) meta1.clone();
-
- assertEquals(meta1, meta2);
- }
-
- @Test
- public void testGetStorageType() {
- assertEquals(StoreType.CSV, meta.getStoreType());
- }
-
- @Test
- public void testGetSchema() {
- Schema schema2 = new Schema();
- schema2.addColumn("name", Type.BLOB);
- schema2.addColumn("addr", Type.TEXT);
-
- assertEquals(schema, schema2);
- }
-
- @Test
- public void testSetSchema() {
- Schema schema2 = new Schema();
- schema2.addColumn("name", Type.BLOB);
- schema2.addColumn("addr", Type.TEXT);
- schema2.addColumn("age", Type.INT4);
-
- assertNotSame(meta.getSchema(), schema2);
- meta.setSchema(schema2);
- assertEquals(meta.getSchema(), schema2);
- }
-
- @Test
- public void testEqualsObject() {
- Schema schema2 = new Schema();
- schema2.addColumn("name", Type.BLOB);
- schema2.addColumn("addr", Type.TEXT);
- TableMeta meta2 = CatalogUtil.newTableMeta(schema2, StoreType.CSV);
-
- assertTrue(meta.equals(meta2));
-
- assertNotSame(meta, meta2);
- }
-
- @Test
- public void testGetProto() {
- Schema schema1 = new Schema();
- schema1.addColumn("name", Type.BLOB);
- schema1.addColumn("addr", Type.TEXT);
- TableMeta meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
-
- TableProto proto = meta1.getProto();
- TableMeta newMeta = new TableMetaImpl(proto);
-
- assertTrue(meta1.equals(newMeta));
- }
-}
[2/5] TAJO-287: Refactor TableDesc, TableMeta, and Fragment. (hyunsik)
Posted by hy...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 522328b..8b50480 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -93,14 +93,10 @@ public class TestPhysicalPlanner {
catalog.createFunction(funcDesc);
}
- Schema schema = new Schema();
- schema.addColumn("name", Type.TEXT);
- schema.addColumn("empId", Type.INT4);
- schema.addColumn("deptName", Type.TEXT);
-
- Schema schema2 = new Schema();
- schema2.addColumn("deptName", Type.TEXT);
- schema2.addColumn("manager", Type.TEXT);
+ Schema employeeSchema = new Schema();
+ employeeSchema.addColumn("name", Type.TEXT);
+ employeeSchema.addColumn("empId", Type.INT4);
+ employeeSchema.addColumn("deptName", Type.TEXT);
Schema scoreSchema = new Schema();
scoreSchema.addColumn("deptName", Type.TEXT);
@@ -108,13 +104,14 @@ public class TestPhysicalPlanner {
scoreSchema.addColumn("score", Type.INT4);
scoreSchema.addColumn("nullable", Type.TEXT);
- TableMeta employeeMeta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema,
+ employeePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(employeeSchema.getColumnNum());
for (int i = 0; i < 100; i++) {
tuple.put(new Datum[] {DatumFactory.createText("name_" + i),
DatumFactory.createInt4(i), DatumFactory.createText("dept_" + i)});
@@ -123,15 +120,15 @@ public class TestPhysicalPlanner {
appender.flush();
appender.close();
- employee = new TableDescImpl("employee", employeeMeta, employeePath);
+ employee = new TableDesc("employee", employeeSchema, employeeMeta, employeePath);
catalog.addTable(employee);
Path scorePath = new Path(testDir, "score");
- TableMeta scoreMeta = CatalogUtil.newTableMeta(scoreSchema, StoreType.CSV, new Options());
- appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreMeta, scorePath);
+ TableMeta scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV, new Options());
+ appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreMeta, scoreSchema, scorePath);
appender.init();
- score = new TableDescImpl("score", scoreMeta, scorePath);
- tuple = new VTuple(score.getMeta().getSchema().getColumnNum());
+ score = new TableDesc("score", scoreSchema, scoreMeta, scorePath);
+ tuple = new VTuple(scoreSchema.getColumnNum());
int m = 0;
for (int i = 1; i <= 5; i++) {
for (int k = 3; k < 5; k++) {
@@ -371,8 +368,7 @@ public class TestPhysicalPlanner {
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(rootNode.getOutSchema(),
- StoreType.CSV);
+ TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.CSV);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -380,7 +376,8 @@ public class TestPhysicalPlanner {
exec.next();
exec.close();
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(outputMeta, ctx.getOutputPath());
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(outputMeta, rootNode.getOutSchema(),
+ ctx.getOutputPath());
scanner.init();
Tuple tuple;
int i = 0;
@@ -411,8 +408,7 @@ public class TestPhysicalPlanner {
LogicalPlan plan = planner.createPlan(context);
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(rootNode.getOutSchema(),
- StoreType.RCFILE);
+ TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.RCFILE);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -420,7 +416,8 @@ public class TestPhysicalPlanner {
exec.next();
exec.close();
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(outputMeta, ctx.getOutputPath());
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(outputMeta, rootNode.getOutSchema(),
+ ctx.getOutputPath());
scanner.init();
Tuple tuple;
int i = 0;
@@ -457,7 +454,7 @@ public class TestPhysicalPlanner {
ctx.setDataChannel(dataChannel);
LogicalNode rootNode = optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(rootNode.getOutSchema(), StoreType.CSV);
+ TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.CSV);
FileSystem fs = sm.getFileSystem();
@@ -474,9 +471,9 @@ public class TestPhysicalPlanner {
Fragment [] fragments = new Fragment[list.length];
int i = 0;
for (FileStatus status : list) {
- fragments[i++] = new Fragment("partition", status.getPath(), outputMeta, 0, status.getLen());
+ fragments[i++] = new Fragment("partition", status.getPath(), 0, status.getLen());
}
- Scanner scanner = new MergeScanner(conf, outputMeta,TUtil.newList(fragments));
+ Scanner scanner = new MergeScanner(conf, outputMeta,rootNode.getOutSchema(), TUtil.newList(fragments));
scanner.init();
Tuple tuple;
@@ -515,8 +512,7 @@ public class TestPhysicalPlanner {
ctx.setDataChannel(dataChannel);
optimizer.optimize(plan);
- TableMeta outputMeta = CatalogUtil.newTableMeta(rootNode.getOutSchema(),
- StoreType.CSV);
+ TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.CSV);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -533,9 +529,9 @@ public class TestPhysicalPlanner {
Fragment [] fragments = new Fragment[list.length];
int i = 0;
for (FileStatus status : list) {
- fragments[i++] = new Fragment("partition", status.getPath(), outputMeta, 0, status.getLen());
+ fragments[i++] = new Fragment("partition", status.getPath(), 0, status.getLen());
}
- Scanner scanner = new MergeScanner(conf, outputMeta,TUtil.newList(fragments));
+ Scanner scanner = new MergeScanner(conf, outputMeta,rootNode.getOutSchema(), TUtil.newList(fragments));
scanner.init();
Tuple tuple;
i = 0;
@@ -798,9 +794,9 @@ public class TestPhysicalPlanner {
keySchema, comp);
reader.open();
Path outputPath = StorageUtil.concatPath(workDir, "output", "output");
- TableMeta meta = CatalogUtil.newTableMeta(rootNode.getOutSchema(), StoreType.CSV, new Options());
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, new Options());
SeekableScanner scanner =
- StorageManagerFactory.getSeekableScanner(conf, meta, outputPath);
+ StorageManagerFactory.getSeekableScanner(conf, meta, employee.getSchema(), outputPath);
scanner.init();
int cnt = 0;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
index f95c3ec..c95881a 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
@@ -19,8 +19,8 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.worker.TaskAttemptContext;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
@@ -37,15 +37,14 @@ import org.apache.tajo.engine.planner.logical.LogicalNode;
import org.apache.tajo.storage.*;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.TUtil;
+import org.apache.tajo.worker.TaskAttemptContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.tajo.LocalTajoTestingUtility;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
// this is not a physical operator in itself, but it uses the HashLeftOuterJoinExec with switched inputs order
public class TestRightOuterHashJoinExec {
@@ -91,12 +90,11 @@ public class TestRightOuterHashJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(dep3Schema,
- StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep3Path = new Path(testDir, "dep3.csv");
- Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Path);
+ Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
- Tuple tuple = new VTuple(dep3Meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(dep3Schema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -106,7 +104,7 @@ public class TestRightOuterHashJoinExec {
appender1.flush();
appender1.close();
- dep3 = CatalogUtil.newTableDesc("dep3", dep3Meta, dep3Path);
+ dep3 = CatalogUtil.newTableDesc("dep3", dep3Schema, dep3Meta, dep3Path);
catalog.addTable(dep3);
//----------------- job3 ------------------------------
@@ -121,12 +119,11 @@ public class TestRightOuterHashJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(job3Schema,
- StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path job3Path = new Path(testDir, "job3.csv");
- Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Path);
+ Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
- Tuple tuple2 = new VTuple(job3Meta.getSchema().getColumnNum());
+ Tuple tuple2 = new VTuple(job3Schema.getColumnNum());
for (int i = 1; i < 4; i++) {
int x = 100 + i;
tuple2.put(new Datum[] { DatumFactory.createInt4(100 + i),
@@ -136,7 +133,7 @@ public class TestRightOuterHashJoinExec {
appender2.flush();
appender2.close();
- job3 = CatalogUtil.newTableDesc("job3", job3Meta, job3Path);
+ job3 = CatalogUtil.newTableDesc("job3", job3Schema, job3Meta, job3Path);
catalog.addTable(job3);
@@ -161,11 +158,11 @@ public class TestRightOuterHashJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(emp3Schema, StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path emp3Path = new Path(testDir, "emp3.csv");
- Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Path);
+ Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
- Tuple tuple3 = new VTuple(emp3Meta.getSchema().getColumnNum());
+ Tuple tuple3 = new VTuple(emp3Schema.getColumnNum());
for (int i = 1; i < 4; i += 2) {
int x = 10 + i;
@@ -200,7 +197,7 @@ public class TestRightOuterHashJoinExec {
appender3.flush();
appender3.close();
- emp3 = CatalogUtil.newTableDesc("emp3", emp3Meta, emp3Path);
+ emp3 = CatalogUtil.newTableDesc("emp3", emp3Schema, emp3Meta, emp3Path);
catalog.addTable(emp3);
analyzer = new SQLAnalyzer();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index 3918734..bf08479 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -94,12 +94,11 @@ public class TestRightOuterMergeJoinExec {
dep3Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep3Meta = CatalogUtil.newTableMeta(dep3Schema,
- StoreType.CSV);
+ TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep3Path = new Path(testDir, "dep3.csv");
- Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Path);
+ Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
- Tuple tuple = new VTuple(dep3Meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(dep3Schema.getColumnNum());
for (int i = 0; i < 10; i++) {
tuple.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -109,7 +108,7 @@ public class TestRightOuterMergeJoinExec {
appender1.flush();
appender1.close();
- dep3 = CatalogUtil.newTableDesc("dep3", dep3Meta, dep3Path);
+ dep3 = CatalogUtil.newTableDesc("dep3", dep3Schema, dep3Meta, dep3Path);
catalog.addTable(dep3);
@@ -133,12 +132,11 @@ public class TestRightOuterMergeJoinExec {
dep4Schema.addColumn("loc_id", Type.INT4);
- TableMeta dep4Meta = CatalogUtil.newTableMeta(dep4Schema,
- StoreType.CSV);
+ TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path dep4Path = new Path(testDir, "dep4.csv");
- Appender appender4 = StorageManagerFactory.getStorageManager(conf).getAppender(dep4Meta, dep4Path);
+ Appender appender4 = StorageManagerFactory.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path);
appender4.init();
- Tuple tuple4 = new VTuple(dep4Meta.getSchema().getColumnNum());
+ Tuple tuple4 = new VTuple(dep4Schema.getColumnNum());
for (int i = 0; i < 11; i++) {
tuple4.put(new Datum[] { DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
@@ -148,7 +146,7 @@ public class TestRightOuterMergeJoinExec {
appender4.flush();
appender4.close();
- dep4 = CatalogUtil.newTableDesc("dep4", dep4Meta, dep4Path);
+ dep4 = CatalogUtil.newTableDesc("dep4", dep4Schema, dep4Meta, dep4Path);
catalog.addTable(dep4);
@@ -165,12 +163,11 @@ public class TestRightOuterMergeJoinExec {
job3Schema.addColumn("job_title", Type.TEXT);
- TableMeta job3Meta = CatalogUtil.newTableMeta(job3Schema,
- StoreType.CSV);
+ TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path job3Path = new Path(testDir, "job3.csv");
- Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Path);
+ Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
- Tuple tuple2 = new VTuple(job3Meta.getSchema().getColumnNum());
+ Tuple tuple2 = new VTuple(job3Schema.getColumnNum());
for (int i = 1; i < 4; i++) {
int x = 100 + i;
tuple2.put(new Datum[] { DatumFactory.createInt4(100 + i),
@@ -180,7 +177,7 @@ public class TestRightOuterMergeJoinExec {
appender2.flush();
appender2.close();
- job3 = CatalogUtil.newTableDesc("job3", job3Meta, job3Path);
+ job3 = CatalogUtil.newTableDesc("job3", job3Schema, job3Meta, job3Path);
catalog.addTable(job3);
@@ -205,11 +202,11 @@ public class TestRightOuterMergeJoinExec {
emp3Schema.addColumn("job_id", Type.INT4);
- TableMeta emp3Meta = CatalogUtil.newTableMeta(emp3Schema, StoreType.CSV);
+ TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path emp3Path = new Path(testDir, "emp3.csv");
- Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Path);
+ Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
- Tuple tuple3 = new VTuple(emp3Meta.getSchema().getColumnNum());
+ Tuple tuple3 = new VTuple(emp3Schema.getColumnNum());
for (int i = 1; i < 4; i += 2) {
int x = 10 + i;
@@ -244,7 +241,7 @@ public class TestRightOuterMergeJoinExec {
appender3.flush();
appender3.close();
- emp3 = CatalogUtil.newTableDesc("emp3", emp3Meta, emp3Path);
+ emp3 = CatalogUtil.newTableDesc("emp3", emp3Schema, emp3Meta, emp3Path);
catalog.addTable(emp3);
//---------------------phone3 --------------------
@@ -257,15 +254,15 @@ public class TestRightOuterMergeJoinExec {
phone3Schema.addColumn("phone_number", Type.TEXT);
- TableMeta phone3Meta = CatalogUtil.newTableMeta(phone3Schema,
- StoreType.CSV);
+ TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path phone3Path = new Path(testDir, "phone3.csv");
- Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Path);
+ Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema,
+ phone3Path);
appender5.init();
appender5.flush();
appender5.close();
- phone3 = CatalogUtil.newTableDesc("phone3", phone3Meta, phone3Path);
+ phone3 = CatalogUtil.newTableDesc("phone3", phone3Schema, phone3Meta, phone3Path);
catalog.addTable(phone3);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index 339da1b..3dc3a4a 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -72,14 +72,14 @@ public class TestSortExec {
schema.addColumn("empId", Type.INT4);
schema.addColumn("deptName", Type.TEXT);
- employeeMeta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
tablePath = StorageUtil.concatPath(workDir, "employee", "table1");
sm.getFileSystem().mkdirs(tablePath.getParent());
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, tablePath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, tablePath);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < 100; i++) {
tuple.put(new Datum[] {
DatumFactory.createInt4(rnd.nextInt(5)),
@@ -90,7 +90,7 @@ public class TestSortExec {
appender.flush();
appender.close();
- TableDesc desc = new TableDescImpl("employee", employeeMeta, tablePath);
+ TableDesc desc = new TableDesc("employee", schema, employeeMeta, tablePath);
catalog.addTable(desc);
analyzer = new SQLAnalyzer();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
index f049185..c005551 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
@@ -64,7 +64,7 @@ public class TestInsertQuery {
res.close();
TableDesc desc = catalog.getTableDesc(tableName);
- assertEquals(5, desc.getMeta().getStat().getNumRows().intValue());
+ assertEquals(5, desc.getStats().getNumRows().intValue());
}
@Test
@@ -80,8 +80,8 @@ public class TestInsertQuery {
res = tpch.execute("insert overwrite into " + tableName + " select l_orderkey from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(tableName);
- assertEquals(5, desc.getMeta().getStat().getNumRows().intValue());
- assertEquals(originalDesc.getMeta().getSchema(), desc.getMeta().getSchema());
+ assertEquals(5, desc.getStats().getNumRows().intValue());
+ assertEquals(originalDesc.getSchema(), desc.getSchema());
}
@Test
@@ -98,7 +98,7 @@ public class TestInsertQuery {
+ " (col1, col3) select l_orderkey, l_quantity from lineitem");
res.close();
TableDesc desc = catalog.getTableDesc(tableName);
- assertEquals(5, desc.getMeta().getStat().getNumRows().intValue());
+ assertEquals(5, desc.getStats().getNumRows().intValue());
res = tpch.execute("select * from " + tableName);
assertTrue(res.next());
@@ -133,7 +133,7 @@ public class TestInsertQuery {
assertFalse(res.next());
res.close();
- assertEquals(originalDesc.getMeta().getSchema(), desc.getMeta().getSchema());
+ assertEquals(originalDesc.getSchema(), desc.getSchema());
}
@Test
@@ -148,7 +148,7 @@ public class TestInsertQuery {
res = tpch.execute("insert overwrite into " + tableName + " select * from lineitem where l_orderkey = 3");
res.close();
TableDesc desc = catalog.getTableDesc(tableName);
- assertEquals(2, desc.getMeta().getStat().getNumRows().intValue());
+ assertEquals(2, desc.getStats().getNumRows().intValue());
}
@Test
@@ -163,7 +163,7 @@ public class TestInsertQuery {
CatalogService catalog = cluster.getMaster().getCatalog();
assertTrue(catalog.existsTable(tableName));
TableDesc orderKeys = catalog.getTableDesc(tableName);
- assertEquals(5, orderKeys.getMeta().getStat().getNumRows().intValue());
+ assertEquals(5, orderKeys.getStats().getNumRows().intValue());
// this query will result in the two rows.
res = tpch.execute(
@@ -173,7 +173,7 @@ public class TestInsertQuery {
assertTrue(catalog.existsTable(tableName));
orderKeys = catalog.getTableDesc(tableName);
- assertEquals(2, orderKeys.getMeta().getStat().getNumRows().intValue());
+ assertEquals(2, orderKeys.getStats().getNumRows().intValue());
}
@Test
@@ -188,7 +188,7 @@ public class TestInsertQuery {
res = tpch.execute("insert overwrite into " + tableName + " select * from lineitem where l_orderkey = 3");
res.close();
TableDesc desc = catalog.getTableDesc(tableName);
- assertEquals(2, desc.getMeta().getStat().getNumRows().intValue());
+ assertEquals(2, desc.getStats().getNumRows().intValue());
}
@Test
@@ -213,7 +213,7 @@ public class TestInsertQuery {
res = tpch.execute("insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_quantity from lineitem where l_orderkey = 3");
res.close();
TableDesc desc = catalog.getTableDesc(tableName);
- assertEquals(2, desc.getMeta().getStat().getNumRows().intValue());
+ assertEquals(2, desc.getStats().getNumRows().intValue());
FileSystem fs = FileSystem.get(tpch.getTestingCluster().getConfiguration());
assertTrue(fs.exists(desc.getPath()));
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestResultSetImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestResultSetImpl.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestResultSetImpl.java
index 643d363..48c424d 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestResultSetImpl.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestResultSetImpl.java
@@ -27,7 +27,7 @@ import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.TpchTestBase;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.ResultSetImpl;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
@@ -51,6 +51,7 @@ public class TestResultSetImpl {
private static TableDesc desc;
private static AbstractStorageManager sm;
private static TableMeta scoreMeta;
+ private static Schema scoreSchema;
@BeforeClass
public static void setup() throws Exception {
@@ -58,15 +59,16 @@ public class TestResultSetImpl {
conf = util.getConfiguration();
sm = StorageManagerFactory.getStorageManager(conf);
- Schema scoreSchema = new Schema();
+ scoreSchema = new Schema();
scoreSchema.addColumn("deptname", Type.TEXT);
scoreSchema.addColumn("score", Type.INT4);
- scoreMeta = CatalogUtil.newTableMeta(scoreSchema, StoreType.CSV);
- TableStat stat = new TableStat();
+ scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV);
+ TableStats stats = new TableStats();
Path p = sm.getTablePath("score");
sm.getFileSystem().mkdirs(p);
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreMeta, new Path(p, "score"));
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreMeta, scoreSchema,
+ new Path(p, "score"));
appender.init();
int deptSize = 100;
int tupleNum = 10000;
@@ -81,13 +83,13 @@ public class TestResultSetImpl {
appender.addTuple(tuple);
}
appender.close();
- stat.setNumRows(tupleNum);
- stat.setNumBytes(written);
- stat.setAvgRows(tupleNum);
- stat.setNumBlocks(1000);
- stat.setNumPartitions(100);
- scoreMeta.setStat(stat);
- desc = new TableDescImpl("score", scoreMeta, p);
+ stats.setNumRows(tupleNum);
+ stats.setNumBytes(written);
+ stats.setAvgRows(tupleNum);
+ stats.setNumBlocks(1000);
+ stats.setNumPartitions(100);
+ desc = new TableDesc("score", scoreSchema, scoreMeta, p);
+ desc.setStats(stats);
}
@AfterClass
@@ -100,7 +102,7 @@ public class TestResultSetImpl {
ResultSetImpl rs = new ResultSetImpl(null, null, conf, desc);
ResultSetMetaData meta = rs.getMetaData();
assertNotNull(meta);
- Schema schema = scoreMeta.getSchema();
+ Schema schema = scoreSchema;
assertEquals(schema.getColumnNum(), meta.getColumnCount());
for (int i = 0; i < meta.getColumnCount(); i++) {
assertEquals(schema.getColumn(i).getColumnName(), meta.getColumnName(i + 1));
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index 0b3a8ce..2547714 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -456,7 +456,7 @@ public class TestSelectQuery {
CatalogService catalog = cluster.getMaster().getCatalog();
assertTrue(catalog.existsTable("orderkeys"));
TableDesc orderKeys = catalog.getTableDesc("orderkeys");
- assertEquals(5, orderKeys.getMeta().getStat().getNumRows().intValue());
+ assertEquals(5, orderKeys.getStats().getNumRows().intValue());
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
index 51dc8fe..c3b5aa3 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
@@ -24,7 +24,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.LogicalOptimizer;
@@ -64,9 +64,9 @@ public class TestExecutionBlockCursor {
tpch.loadSchemas();
tpch.loadOutSchema();
for (String table : tpch.getTableNames()) {
- TableMeta m = CatalogUtil.newTableMeta(tpch.getSchema(table), CatalogProtos.StoreType.CSV);
- m.setStat(new TableStat());
- TableDesc d = CatalogUtil.newTableDesc(table, m, CommonTestingUtil.getTestDir());
+ TableMeta m = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
+ TableDesc d = CatalogUtil.newTableDesc(table, tpch.getSchema(table), m, CommonTestingUtil.getTestDir());
+ d.setStats(new TableStats());
catalog.addTable(d);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
index e85b54a..ca1247a 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
@@ -24,7 +24,7 @@ import org.apache.tajo.algebra.Expr;
import org.apache.tajo.benchmark.TPCH;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.LogicalOptimizer;
import org.apache.tajo.engine.planner.LogicalPlan;
@@ -72,12 +72,11 @@ public class TestGlobalPlanner {
tpch.loadSchemas();
tpch.loadOutSchema();
for (int i = 0; i < tables.length; i++) {
- TableMeta m = CatalogUtil.newTableMeta(tpch.getSchema(tables[i]), CatalogProtos.StoreType.CSV);
- TableStat stat = new TableStat();
- stat.setNumBytes(volumes[i]);
- m.setStat(stat);
-
- TableDesc d = CatalogUtil.newTableDesc(tables[i], m, CommonTestingUtil.getTestDir());
+ TableMeta m = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
+ TableStats stats = new TableStats();
+ stats.setNumBytes(volumes[i]);
+ TableDesc d = CatalogUtil.newTableDesc(tables[i], tpch.getSchema(tables[i]), m, CommonTestingUtil.getTestDir());
+ d.setStats(stats);
catalog.addTable(d);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFragment.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFragment.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFragment.java
index 5ff6844..630acd1 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFragment.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestFragment.java
@@ -20,14 +20,11 @@ package org.apache.tajo.storage;
import com.google.common.collect.Sets;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.util.CommonTestingUtil;
import org.junit.Before;
import org.junit.Test;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.common.TajoDataTypes.Type;
import java.util.Arrays;
import java.util.SortedSet;
@@ -37,7 +34,6 @@ import static org.junit.Assert.assertTrue;
public class TestFragment {
private Schema schema1;
- private TableMeta meta1;
private Path path;
@Before
@@ -45,47 +41,29 @@ public class TestFragment {
schema1 = new Schema();
schema1.addColumn("id", Type.INT4);
schema1.addColumn("name", Type.TEXT);
- meta1 = CatalogUtil.newTableMeta(schema1, StoreType.CSV);
path = CommonTestingUtil.getTestDir();
}
@Test
- public final void testGetAndSetFields() {
- Fragment fragment1 = new Fragment("table1_1", new Path(path, "table0"), meta1, 0, 500);
+ public final void testGetAndSetFields() {
+ Fragment fragment1 = new Fragment("table1_1", new Path(path, "table0"), 0, 500);
fragment1.setDistCached();
assertEquals("table1_1", fragment1.getName());
assertEquals(new Path(path, "table0"), fragment1.getPath());
- assertEquals(meta1.getStoreType(), fragment1.getMeta().getStoreType());
- assertEquals(meta1.getSchema().getColumnNum(),
- fragment1.getMeta().getSchema().getColumnNum());
assertTrue(fragment1.isDistCached());
- for(int i=0; i < meta1.getSchema().getColumnNum(); i++) {
- assertEquals(meta1.getSchema().getColumn(i).getColumnName(),
- fragment1.getMeta().getSchema().getColumn(i).getColumnName());
- assertEquals(meta1.getSchema().getColumn(i).getDataType(),
- fragment1.getMeta().getSchema().getColumn(i).getDataType());
- }
assertTrue(0 == fragment1.getStartOffset());
assertTrue(500 == fragment1.getLength());
}
@Test
public final void testTabletTabletProto() {
- Fragment fragment0 = new Fragment("table1_1", new Path(path, "table0"), meta1, 0, 500);
-
+ Fragment fragment0 = new Fragment("table1_1", new Path(path, "table0"), 0, 500);
+ fragment0.setDistCached();
Fragment fragment1 = new Fragment(fragment0.getProto());
assertEquals("table1_1", fragment1.getName());
assertEquals(new Path(path, "table0"), fragment1.getPath());
- assertEquals(meta1.getStoreType(), fragment1.getMeta().getStoreType());
- assertEquals(meta1.getSchema().getColumnNum(),
- fragment1.getMeta().getSchema().getColumnNum());
- for(int i=0; i < meta1.getSchema().getColumnNum(); i++) {
- assertEquals(meta1.getSchema().getColumn(i).getColumnName(),
- fragment1.getMeta().getSchema().getColumn(i).getColumnName());
- assertEquals(meta1.getSchema().getColumn(i).getDataType(),
- fragment1.getMeta().getSchema().getColumn(i).getDataType());
- }
+ assertTrue(fragment1.isDistCached());
assertTrue(0 == fragment1.getStartOffset());
assertTrue(500 == fragment1.getLength());
}
@@ -95,7 +73,7 @@ public class TestFragment {
final int num = 10;
Fragment [] tablets = new Fragment[num];
for (int i = num - 1; i >= 0; i--) {
- tablets[i] = new Fragment("tablet1_"+i, new Path(path, "tablet0"), meta1, i * 500, (i+1) * 500);
+ tablets[i] = new Fragment("tablet1_"+i, new Path(path, "tablet0"), i * 500, (i+1) * 500);
}
Arrays.sort(tablets);
@@ -110,7 +88,7 @@ public class TestFragment {
final int num = 1860;
Fragment [] tablets = new Fragment[num];
for (int i = num - 1; i >= 0; i--) {
- tablets[i] = new Fragment("tablet1_"+i, new Path(path, "tablet0"), meta1, (long)i * 6553500, (long)(i+1) * 6553500);
+ tablets[i] = new Fragment("tablet1_"+i, new Path(path, "tablet0"), (long)i * 6553500, (long)(i+1) * 6553500);
}
SortedSet sortedSet = Sets.newTreeSet();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
index 147fe1d..2678a54 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
@@ -27,10 +27,9 @@ import org.apache.tajo.TpchTestBase;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.TableMetaImpl;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
@@ -68,7 +67,7 @@ public class TestRowFile {
schema.addColumn("age", Type.INT8);
schema.addColumn("description", Type.TEXT);
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.ROWFILE);
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.ROWFILE);
AbstractStorageManager sm = StorageManagerFactory.getStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR)));
@@ -80,7 +79,7 @@ public class TestRowFile {
FileUtil.writeProto(fs, metaPath, meta.getProto());
- Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, dataPath);
+ Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, dataPath);
appender.enableStats();
appender.init();
@@ -100,18 +99,18 @@ public class TestRowFile {
}
appender.close();
- TableStat stat = appender.getStats();
+ TableStats stat = appender.getStats();
assertEquals(tupleNum, stat.getNumRows().longValue());
FileStatus file = fs.getFileStatus(dataPath);
TableProto proto = (TableProto) FileUtil.loadProto(
cluster.getDefaultFileSystem(), metaPath, TableProto.getDefaultInstance());
- meta = new TableMetaImpl(proto);
- Fragment fragment = new Fragment("test.tbl", dataPath, meta, 0, file.getLen());
+ meta = new TableMeta(proto);
+ Fragment fragment = new Fragment("test.tbl", dataPath, 0, file.getLen());
int tupleCnt = 0;
start = System.currentTimeMillis();
- Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, fragment);
+ Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment);
scanner.init();
while ((tuple=scanner.next()) != null) {
tupleCnt++;
@@ -125,8 +124,8 @@ public class TestRowFile {
long fileLen = file.getLen()/13;
for (int i = 0; i < 13; i++) {
- fragment = new Fragment("test.tbl", dataPath, meta, fileStart, fileLen);
- scanner = new RowFile.RowFileScanner(conf, meta, fragment);
+ fragment = new Fragment("test.tbl", dataPath, fileStart, fileLen);
+ scanner = new RowFile.RowFileScanner(conf, meta, schema, fragment);
scanner.init();
while ((tuple=scanner.next()) != null) {
if (!idSet.remove(tuple.get(0).asInt4())) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
index 307475a..3b568fc 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
@@ -100,14 +100,14 @@ public class TestRangeRetrieverHandler {
Tuple firstTuple = null;
Tuple lastTuple;
- TableMeta employeeMeta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tableDir = StorageUtil.concatPath(testDir, "testGet", "table.csv");
fs.mkdirs(tableDir.getParent());
- Appender appender = sm.getAppender(employeeMeta, tableDir);
+ Appender appender = sm.getAppender(employeeMeta, schema, tableDir);
appender.init();
- Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < TEST_TUPLE; i++) {
tuple.put(
new Datum[] {
@@ -124,7 +124,7 @@ public class TestRangeRetrieverHandler {
appender.flush();
appender.close();
- TableDesc employee = new TableDescImpl("employee", employeeMeta, tableDir);
+ TableDesc employee = new TableDesc("employee", schema, employeeMeta, tableDir);
catalog.addTable(employee);
Fragment[] frags = StorageManager.splitNG(conf, "employee", employeeMeta, tableDir, Integer.MAX_VALUE);
@@ -158,7 +158,7 @@ public class TestRangeRetrieverHandler {
new Path(testDir, "output/index"), keySchema, comp);
reader.open();
- SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, employeeMeta,
+ SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, employeeMeta, schema,
StorageUtil.concatPath(testDir, "output", "output"));
scanner.init();
@@ -215,12 +215,12 @@ public class TestRangeRetrieverHandler {
Tuple firstTuple = null;
Tuple lastTuple;
- TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV);
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
Path tablePath = StorageUtil.concatPath(testDir, "testGetFromDescendingOrder", "table.csv");
fs.mkdirs(tablePath.getParent());
- Appender appender = sm.getAppender(meta, tablePath);
+ Appender appender = sm.getAppender(meta, schema, tablePath);
appender.init();
- Tuple tuple = new VTuple(meta.getSchema().getColumnNum());
+ Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = (TEST_TUPLE - 1); i >= 0 ; i--) {
tuple.put(
new Datum[] {
@@ -237,7 +237,7 @@ public class TestRangeRetrieverHandler {
appender.flush();
appender.close();
- TableDesc employee = new TableDescImpl("employee", meta, tablePath);
+ TableDesc employee = new TableDesc("employee", schema, meta, tablePath);
catalog.addTable(employee);
Fragment[] frags = sm.splitNG(conf, "employee", meta, tablePath, Integer.MAX_VALUE);
@@ -269,7 +269,7 @@ public class TestRangeRetrieverHandler {
BSTIndex.BSTIndexReader reader = bst.getIndexReader(
new Path(testDir, "output/index"), keySchema, comp);
reader.open();
- SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta,
+ SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema,
StorageUtil.concatPath(testDir, "output", "output"));
scanner.init();
int cnt = 0;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
index ef02d2b..0e695a3 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
@@ -28,15 +28,16 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.TableMetaImpl;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.annotation.ForSplitableStore;
import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.FileUtil;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -67,8 +68,9 @@ public abstract class AbstractStorageManager {
private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
new ConcurrentHashMap<Class<?>, Constructor<?>>();
- public abstract Scanner getScanner(TableMeta meta, Fragment fragment,
- Schema target) throws IOException;
+ public abstract Class<? extends Scanner> getScannerClass(CatalogProtos.StoreType storeType) throws IOException;
+
+ public abstract Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException;
protected AbstractStorageManager(TajoConf conf) throws IOException {
this.conf = conf;
@@ -80,17 +82,17 @@ public abstract class AbstractStorageManager {
LOG.warn("does not support block metadata. ('dfs.datanode.hdfs-blocks-metadata.enabled')");
}
- public Scanner getScanner(TableMeta meta, Path path)
+ public Scanner getScanner(TableMeta meta, Schema schema, Path path)
throws IOException {
FileSystem fs = path.getFileSystem(conf);
FileStatus status = fs.getFileStatus(path);
- Fragment fragment = new Fragment(path.getName(), path, meta, 0, status.getLen());
- return getScanner(meta, fragment);
+ Fragment fragment = new Fragment(path.getName(), path, 0, status.getLen());
+ return getScanner(meta, schema, fragment);
}
- public Scanner getScanner(TableMeta meta, Fragment fragment)
+ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment)
throws IOException {
- return getScanner(meta, fragment, meta.getSchema());
+ return getScanner(meta, schema, fragment, schema);
}
public FileSystem getFileSystem() {
@@ -129,7 +131,7 @@ public abstract class AbstractStorageManager {
return new Path(tableBaseDir, tableName);
}
- public Appender getAppender(TableMeta meta, Path path)
+ public Appender getAppender(TableMeta meta, Schema schema, Path path)
throws IOException {
Appender appender;
@@ -149,7 +151,7 @@ public abstract class AbstractStorageManager {
throw new IOException("Unknown Storage Type: " + meta.getStoreType());
}
- appender = newAppenderInstance(appenderClass, conf, meta, path);
+ appender = newAppenderInstance(appenderClass, conf, meta, schema, path);
return appender;
}
@@ -168,7 +170,7 @@ public abstract class AbstractStorageManager {
CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) FileUtil.loadProto(tableMetaIn,
CatalogProtos.TableProto.getDefaultInstance());
- meta = new TableMetaImpl(tableProto);
+ meta = new TableMeta(tableProto);
return meta;
}
@@ -191,7 +193,7 @@ public abstract class AbstractStorageManager {
FileStatus[] fileLists = fs.listStatus(tablePath);
for (FileStatus file : fileLists) {
- tablet = new Fragment(tablePath.getName(), file.getPath(), meta, 0, file.getLen());
+ tablet = new Fragment(tablePath.getName(), file.getPath(), 0, file.getLen());
listTablets.add(tablet);
}
@@ -225,14 +227,14 @@ public abstract class AbstractStorageManager {
long start = 0;
if (remainFileSize > defaultBlockSize) {
while (remainFileSize > defaultBlockSize) {
- tablet = new Fragment(tableName, file.getPath(), meta, start, defaultBlockSize);
+ tablet = new Fragment(tableName, file.getPath(), start, defaultBlockSize);
listTablets.add(tablet);
start += defaultBlockSize;
remainFileSize -= defaultBlockSize;
}
- listTablets.add(new Fragment(tableName, file.getPath(), meta, start, remainFileSize));
+ listTablets.add(new Fragment(tableName, file.getPath(), start, remainFileSize));
} else {
- listTablets.add(new Fragment(tableName, file.getPath(), meta, 0, remainFileSize));
+ listTablets.add(new Fragment(tableName, file.getPath(), 0, remainFileSize));
}
}
@@ -257,14 +259,14 @@ public abstract class AbstractStorageManager {
long start = 0;
if (remainFileSize > defaultBlockSize) {
while (remainFileSize > defaultBlockSize) {
- tablet = new Fragment(tableName, file.getPath(), meta, start, defaultBlockSize);
+ tablet = new Fragment(tableName, file.getPath(), start, defaultBlockSize);
listTablets.add(tablet);
start += defaultBlockSize;
remainFileSize -= defaultBlockSize;
}
- listTablets.add(new Fragment(tableName, file.getPath(), meta, start, remainFileSize));
+ listTablets.add(new Fragment(tableName, file.getPath(), start, remainFileSize));
} else {
- listTablets.add(new Fragment(tableName, file.getPath(), meta, 0, remainFileSize));
+ listTablets.add(new Fragment(tableName, file.getPath(), 0, remainFileSize));
}
}
@@ -389,14 +391,29 @@ public abstract class AbstractStorageManager {
* <code>false</code> to ensure that individual input files are never split-up
* so that Mappers process entire files.
*
+ *
* @param filename the file name to check
* @return is this file isSplittable?
*/
- protected boolean isSplittable(TableMeta meta, Path filename) throws IOException {
- Scanner scanner = getScanner(meta, filename);
+ protected boolean isSplittable(TableMeta meta, Schema schema, Path filename) throws IOException {
+ Scanner scanner = getScanner(meta, schema, filename);
return scanner.isSplittable();
}
+ protected boolean isSplittable(CatalogProtos.StoreType storeType) throws IOException {
+ Method[] methods = getScannerClass(storeType).getMethods();
+
+ for (Method method : methods) {
+ ForSplitableStore annos = method.getAnnotation(ForSplitableStore.class);
+ if (annos != null) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+
@Deprecated
protected long computeSplitSize(long blockSize, long minSize,
long maxSize) {
@@ -428,12 +445,12 @@ public abstract class AbstractStorageManager {
* by sub-classes to make sub-types
*/
protected Fragment makeSplit(String fragmentId, TableMeta meta, Path file, long start, long length) {
- return new Fragment(fragmentId, file, meta, start, length);
+ return new Fragment(fragmentId, file, start, length);
}
protected Fragment makeSplit(String fragmentId, TableMeta meta, Path file, BlockLocation blockLocation,
int[] diskIds) throws IOException {
- return new Fragment(fragmentId, file, meta, blockLocation, diskIds);
+ return new Fragment(fragmentId, file, blockLocation, diskIds);
}
// for Non Splittable. eg, compressed gzip TextFile
@@ -468,7 +485,7 @@ public abstract class AbstractStorageManager {
hosts[i] = entry.getKey();
hostsBlockCount[i] = entry.getValue();
}
- return new Fragment(fragmentId, file, meta, start, length, hosts, hostsBlockCount);
+ return new Fragment(fragmentId, file, start, length, hosts, hostsBlockCount);
}
/**
@@ -544,7 +561,7 @@ public abstract class AbstractStorageManager {
*
* @throws IOException
*/
- public List<Fragment> getSplits(String tableName, TableMeta meta, Path inputPath) throws IOException {
+ public List<Fragment> getSplits(String tableName, TableMeta meta, Schema schema, Path inputPath) throws IOException {
// generate splits'
List<Fragment> splits = new ArrayList<Fragment>();
@@ -555,7 +572,7 @@ public abstract class AbstractStorageManager {
long length = file.getLen();
if (length > 0) {
BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
- boolean splittable = isSplittable(meta, path);
+ boolean splittable = isSplittable(meta, schema, inputPath);
if (blocksMetadataEnabled && fs instanceof DistributedFileSystem) {
// supported disk volume
BlockStorageLocation[] blockStorageLocations = ((DistributedFileSystem) fs)
@@ -597,19 +614,21 @@ public abstract class AbstractStorageManager {
private static final Class<?>[] DEFAULT_SCANNER_PARAMS = {
Configuration.class,
TableMeta.class,
+ Schema.class,
Fragment.class
};
private static final Class<?>[] DEFAULT_APPENDER_PARAMS = {
Configuration.class,
TableMeta.class,
+ Schema.class,
Path.class
};
/**
* create a scanner instance.
*/
- public static <T> T newScannerInstance(Class<T> theClass, Configuration conf, TableMeta meta,
+ public static <T> T newScannerInstance(Class<T> theClass, Configuration conf, TableMeta meta, Schema schema,
Fragment fragment) {
T result;
try {
@@ -619,7 +638,7 @@ public abstract class AbstractStorageManager {
meth.setAccessible(true);
CONSTRUCTOR_CACHE.put(theClass, meth);
}
- result = meth.newInstance(new Object[]{conf, meta, fragment});
+ result = meth.newInstance(new Object[]{conf, meta, schema, fragment});
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -630,7 +649,7 @@ public abstract class AbstractStorageManager {
/**
* create a scanner instance.
*/
- public static <T> T newAppenderInstance(Class<T> theClass, Configuration conf, TableMeta meta,
+ public static <T> T newAppenderInstance(Class<T> theClass, Configuration conf, TableMeta meta, Schema schema,
Path path) {
T result;
try {
@@ -640,7 +659,7 @@ public abstract class AbstractStorageManager {
meth.setAccessible(true);
CONSTRUCTOR_CACHE.put(theClass, meth);
}
- result = meth.newInstance(new Object[]{conf, meta, path});
+ result = meth.newInstance(new Object[]{conf, meta, schema, path});
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Appender.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Appender.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Appender.java
index 6720614..ed6ea34 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Appender.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Appender.java
@@ -18,7 +18,7 @@
package org.apache.tajo.storage;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import java.io.Closeable;
import java.io.IOException;
@@ -35,5 +35,5 @@ public interface Appender extends Closeable {
void enableStats();
- TableStat getStats();
+ TableStats getStats();
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 550148c..5ac989d 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.io.compress.*;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.datum.CharDatum;
import org.apache.tajo.datum.Datum;
@@ -39,6 +39,7 @@ import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.datum.ProtobufDatum;
import org.apache.tajo.datum.protobuf.ProtobufJsonFormat;
import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.storage.annotation.ForSplitableStore;
import org.apache.tajo.storage.compress.CodecPool;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
import org.apache.tajo.util.Bytes;
@@ -74,12 +75,11 @@ public class CSVFile {
private byte[] nullChars;
private ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
- public CSVAppender(Configuration conf, final TableMeta meta,
- final Path path) throws IOException {
- super(conf, meta, path);
+ public CSVAppender(Configuration conf, final TableMeta meta, final Schema schema, final Path path) throws IOException {
+ super(conf, meta, schema, path);
this.fs = path.getFileSystem(conf);
this.meta = meta;
- this.schema = meta.getSchema();
+ this.schema = schema;
this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(DELIMITER, DELIMITER_DEFAULT)).charAt(0);
String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(NULL));
@@ -298,7 +298,7 @@ public class CSVFile {
}
@Override
- public TableStat getStats() {
+ public TableStats getStats() {
if (enabledStats) {
return stats.getTableStat();
} else {
@@ -315,10 +315,11 @@ public class CSVFile {
}
}
+ @ForSplitableStore
public static class CSVScanner extends FileScanner implements SeekableScanner {
- public CSVScanner(Configuration conf, final TableMeta meta,
- final Fragment fragment) throws IOException {
- super(conf, meta, fragment);
+ public CSVScanner(Configuration conf, final TableMeta meta, final Schema schema, final Fragment fragment)
+ throws IOException {
+ super(conf, meta, schema, fragment);
factory = new CompressionCodecFactory(conf);
codec = factory.getCodec(fragment.getPath());
if (isCompress() && !(codec instanceof SplittableCompressionCodec)) {
@@ -327,10 +328,10 @@ public class CSVFile {
// Buffer size, Delimiter
this.bufSize = DEFAULT_BUFFER_SIZE;
- String delim = fragment.getMeta().getOption(DELIMITER, DELIMITER_DEFAULT);
+ String delim = meta.getOption(DELIMITER, DELIMITER_DEFAULT);
this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0);
- String nullCharacters = StringEscapeUtils.unescapeJava(fragment.getMeta().getOption(NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(NULL));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
index ca7c898..55a9bf3 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
@@ -35,10 +35,10 @@ public abstract class FileAppender implements Appender {
protected boolean enabledStats;
- public FileAppender(Configuration conf, TableMeta meta, Path path) {
+ public FileAppender(Configuration conf, TableMeta meta, Schema schema, Path path) {
this.conf = conf;
this.meta = meta;
- this.schema = meta.getSchema();
+ this.schema = schema;
this.path = path;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileScanner.java
index 005879a..9c22b4f 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/FileScanner.java
@@ -35,10 +35,10 @@ public abstract class FileScanner implements Scanner {
protected Column [] targets;
- public FileScanner(Configuration conf, final TableMeta meta, final Fragment fragment) {
+ public FileScanner(Configuration conf, final TableMeta meta, final Schema schema, final Fragment fragment) {
this.conf = conf;
this.meta = meta;
- this.schema = meta.getSchema();
+ this.schema = schema;
this.fragment = fragment;
this.columnNum = this.schema.getColumnNum();
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Fragment.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Fragment.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Fragment.java
index 6c31247..f40c9a7 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Fragment.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Fragment.java
@@ -19,15 +19,10 @@
package org.apache.tajo.storage;
import com.google.common.base.Objects;
-import com.google.gson.Gson;
import com.google.gson.annotations.Expose;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
-import org.apache.tajo.json.GsonObject;
-import org.apache.tajo.storage.json.StorageGsonHelper;
import org.apache.tajo.util.TUtil;
import java.io.IOException;
@@ -35,12 +30,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-public class Fragment implements TableDesc, Comparable<Fragment>, SchemaObject, GsonObject {
+public class Fragment implements Comparable<Fragment> {
protected FragmentProto.Builder builder = null;
@Expose private String tableName; // required
@Expose private Path uri; // required
- @Expose private TableMeta meta; // required
@Expose private Long startOffset; // required
@Expose private Long length; // required
@Expose private boolean distCached = false; // optional
@@ -53,47 +47,35 @@ public class Fragment implements TableDesc, Comparable<Fragment>, SchemaObject,
builder = FragmentProto.newBuilder();
}
- public Fragment(String tableName, Path uri, TableMeta meta, BlockLocation blockLocation, int[] diskIds) throws IOException {
+ public Fragment(String tableName, Path uri, BlockLocation blockLocation, int[] diskIds)
+ throws IOException {
this();
- //TableMeta newMeta = new TableMetaImpl(meta.getProto());
- TableMeta newMeta = meta;
- SchemaProto newSchemaProto = CatalogUtil.getQualfiedSchema(tableName, meta
- .getSchema().getProto());
- newMeta.setSchema(new Schema(newSchemaProto));
- this.set(tableName, uri, newMeta, blockLocation.getOffset(), blockLocation.getLength(),
+ this.set(tableName, uri, blockLocation.getOffset(), blockLocation.getLength(),
blockLocation.getHosts(), diskIds);
}
// Non splittable
- public Fragment(String tableName, Path uri, TableMeta meta, long start, long length, String[] hosts, int[] hostsBlockCount) {
+ public Fragment(String tableName, Path uri, long start, long length, String[] hosts,
+ int[] hostsBlockCount) {
this();
- TableMeta newMeta = new TableMetaImpl(meta.getProto());
- SchemaProto newSchemaProto = CatalogUtil.getQualfiedSchema(tableName, meta
- .getSchema().getProto());
- newMeta.setSchema(new Schema(newSchemaProto));
- this.set(tableName, uri, newMeta, start, length, null, null);
+ this.set(tableName, uri, start, length, null, null);
this.hosts = hosts;
this.hostsBlockCount = hostsBlockCount;
}
- public Fragment(String fragmentId, Path path, TableMeta meta, long start, long length) {
+ public Fragment(String fragmentId, Path path, long start, long length) {
this();
- TableMeta newMeta = new TableMetaImpl(meta.getProto());
- SchemaProto newSchemaProto = CatalogUtil.getQualfiedSchema(fragmentId, meta
- .getSchema().getProto());
- newMeta.setSchema(new Schema(newSchemaProto));
- this.set(fragmentId, path, newMeta, start, length, null, null);
+ this.set(fragmentId, path, start, length, null, null);
}
public Fragment(FragmentProto proto) {
this();
- TableMeta newMeta = new TableMetaImpl(proto.getMeta());
int[] diskIds = new int[proto.getDiskIdsList().size()];
int i = 0;
for(Integer eachValue: proto.getDiskIdsList()) {
diskIds[i++] = eachValue;
}
- this.set(proto.getId(), new Path(proto.getPath()), newMeta,
+ this.set(proto.getId(), new Path(proto.getPath()),
proto.getStartOffset(), proto.getLength(),
proto.getHostsList().toArray(new String[]{}),
diskIds);
@@ -102,11 +84,10 @@ public class Fragment implements TableDesc, Comparable<Fragment>, SchemaObject,
}
}
- private void set(String tableName, Path path, TableMeta meta, long start,
+ private void set(String tableName, Path path, long start,
long length, String[] hosts, int[] diskIds) {
this.tableName = tableName;
this.uri = path;
- this.meta = meta;
this.startOffset = start;
this.length = length;
this.hosts = hosts;
@@ -152,33 +133,17 @@ public class Fragment implements TableDesc, Comparable<Fragment>, SchemaObject,
return this.tableName;
}
- @Override
public void setName(String tableName) {
this.tableName = tableName;
}
-
- @Override
+
public Path getPath() {
return this.uri;
}
- @Override
public void setPath(Path path) {
this.uri = path;
}
-
- public Schema getSchema() {
- return getMeta().getSchema();
- }
-
- public TableMeta getMeta() {
- return this.meta;
- }
-
- @Override
- public void setMeta(TableMeta meta) {
- this.meta = meta;
- }
public Long getStartOffset() {
return this.startOffset;
@@ -243,7 +208,6 @@ public class Fragment implements TableDesc, Comparable<Fragment>, SchemaObject,
frag.builder = FragmentProto.newBuilder();
frag.tableName = tableName;
frag.uri = uri;
- frag.meta = (TableMeta) (meta != null ? meta.clone() : null);
frag.distCached = distCached;
frag.diskIds = diskIds;
frag.hosts = hosts;
@@ -259,14 +223,12 @@ public class Fragment implements TableDesc, Comparable<Fragment>, SchemaObject,
+ getLength() + ", \"distCached\": " + distCached + "}" ;
}
- @Override
public FragmentProto getProto() {
if (builder == null) {
builder = FragmentProto.newBuilder();
}
builder.setId(this.tableName);
builder.setStartOffset(this.startOffset);
- builder.setMeta(meta.getProto());
builder.setLength(this.length);
builder.setPath(this.uri.toString());
builder.setDistCached(this.distCached);
@@ -284,10 +246,4 @@ public class Fragment implements TableDesc, Comparable<Fragment>, SchemaObject,
return builder.build();
}
-
- @Override
- public String toJson() {
- Gson gson = StorageGsonHelper.getInstance();
- return gson.toJson(this, TableDesc.class);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
index 582c64f..b392e4e 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
@@ -33,15 +33,17 @@ import java.util.List;
public class MergeScanner implements Scanner {
private Configuration conf;
private TableMeta meta;
+ private Schema schema;
private List<Fragment> fragments;
private Iterator<Fragment> iterator;
private Fragment currentFragment;
private Scanner currentScanner;
private Tuple tuple;
- public MergeScanner(Configuration conf, TableMeta meta, Collection<Fragment> fragments) {
+ public MergeScanner(Configuration conf, TableMeta meta, Schema schema, Collection<Fragment> fragments) {
this.conf = conf;
this.meta = meta;
+ this.schema = schema;
this.fragments = new ArrayList<Fragment>(fragments);
iterator = this.fragments.iterator();
}
@@ -62,7 +64,8 @@ public class MergeScanner implements Scanner {
currentScanner.close();
}
currentFragment = iterator.next();
- currentScanner = StorageManagerFactory.getStorageManager((TajoConf)conf).getScanner(meta, currentFragment);
+ currentScanner = StorageManagerFactory.getStorageManager((TajoConf)conf).getScanner(meta, schema,
+ currentFragment);
currentScanner.init();
return currentScanner.next();
} else {
@@ -75,7 +78,8 @@ public class MergeScanner implements Scanner {
iterator = fragments.iterator();
if (iterator.hasNext()) {
currentFragment = iterator.next();
- currentScanner = StorageManagerFactory.getStorageManager((TajoConf)conf).getScanner(meta, currentFragment);
+ currentScanner = StorageManagerFactory.getStorageManager((TajoConf)conf).getScanner(meta, schema,
+ currentFragment);
}
}
@@ -110,7 +114,7 @@ public class MergeScanner implements Scanner {
@Override
public Schema getSchema() {
- return meta.getSchema();
+ return schema;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index d89a2fc..0eb39c1 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -24,8 +24,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
@@ -57,15 +58,15 @@ public class RawFile {
private boolean eof = false;
private long fileSize;
- public RawFileScanner(Configuration conf, TableMeta meta, Path path) throws IOException {
- super(conf, meta, null);
+ public RawFileScanner(Configuration conf, TableMeta meta, Schema schema, Path path) throws IOException {
+ super(conf, meta, schema, null);
this.path = path;
init();
}
@SuppressWarnings("unused")
- public RawFileScanner(Configuration conf, TableMeta meta, Fragment fragment) throws IOException {
- this(conf, meta, fragment.getPath());
+ public RawFileScanner(Configuration conf, TableMeta meta, Schema schema, Fragment fragment) throws IOException {
+ this(conf, meta, schema, fragment.getPath());
}
public void init() throws IOException {
@@ -308,8 +309,8 @@ public class RawFile {
private TableStatistics stats;
- public RawFileAppender(Configuration conf, TableMeta meta, Path path) throws IOException {
- super(conf, meta, path);
+ public RawFileAppender(Configuration conf, TableMeta meta, Schema schema, Path path) throws IOException {
+ super(conf, meta, schema, path);
}
public void init() throws IOException {
@@ -512,7 +513,7 @@ public class RawFile {
}
@Override
- public TableStat getStats() {
+ public TableStats getStats() {
if (enabledStats) {
return stats.getTableStat();
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
index d2b6d8e..f379918 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/RowFile.java
@@ -26,11 +26,13 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.storage.annotation.ForSplitableStore;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
import org.apache.tajo.util.BitArray;
@@ -50,6 +52,7 @@ public class RowFile {
private final static int DEFAULT_BUFFER_SIZE = 65535;
public static int SYNC_INTERVAL;
+ @ForSplitableStore
public static class RowFileScanner extends FileScanner {
private FileSystem fs;
private FSDataInputStream in;
@@ -65,9 +68,9 @@ public class RowFile {
private int numBitsOfNullFlags;
private long bufferStartPos;
- public RowFileScanner(Configuration conf, final TableMeta meta,
- final Fragment fragment) throws IOException {
- super(conf, meta, fragment);
+ public RowFileScanner(Configuration conf, final TableMeta meta, final Schema schema, final Fragment fragment)
+ throws IOException {
+ super(conf, meta, schema, fragment);
SYNC_INTERVAL =
conf.getInt(ConfVars.RAWFILE_SYNC_INTERVAL.varname,
@@ -323,9 +326,9 @@ public class RowFile {
// statistics
private TableStatistics stats;
- public RowFileAppender(Configuration conf, final TableMeta meta, final Path path)
+ public RowFileAppender(Configuration conf, final TableMeta meta, final Schema schema, final Path path)
throws IOException {
- super(conf, meta, path);
+ super(conf, meta, schema, path);
}
public void init() throws IOException {
@@ -495,7 +498,7 @@ public class RowFile {
}
@Override
- public TableStat getStats() {
+ public TableStats getStats() {
if (enabledStats) {
return stats.getTableStat();
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
index 9907591..6dc13e5 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -20,6 +20,7 @@ package org.apache.tajo.storage;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
import java.io.IOException;
@@ -34,27 +35,28 @@ public class StorageManager extends AbstractStorageManager {
}
@Override
- public Scanner getScanner(TableMeta meta, Fragment fragment,
- Schema target) throws IOException {
- Scanner scanner;
-
- Class<? extends Scanner> scannerClass;
-
- String handlerName = meta.getStoreType().name().toLowerCase();
- scannerClass = SCANNER_HANDLER_CACHE.get(handlerName);
+ public Class<? extends Scanner> getScannerClass(CatalogProtos.StoreType storeType) throws IOException {
+ String handlerName = storeType.name().toLowerCase();
+ Class<? extends Scanner> scannerClass = SCANNER_HANDLER_CACHE.get(handlerName);
if (scannerClass == null) {
scannerClass = conf.getClass(
- String.format("tajo.storage.scanner-handler.%s.class",
- meta.getStoreType().name().toLowerCase()), null,
- Scanner.class);
+ String.format("tajo.storage.scanner-handler.%s.class",storeType.name().toLowerCase()), null, Scanner.class);
SCANNER_HANDLER_CACHE.put(handlerName, scannerClass);
}
if (scannerClass == null) {
- throw new IOException("Unknown Storage Type: " + meta.getStoreType());
+ throw new IOException("Unknown Storage Type: " + storeType.name());
}
- scanner = newScannerInstance(scannerClass, conf, meta, fragment);
+ return scannerClass;
+ }
+
+ @Override
+ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException {
+ Scanner scanner;
+
+ Class<? extends Scanner> scannerClass = getScannerClass(meta.getStoreType());
+ scanner = newScannerInstance(scannerClass, conf, meta, schema, fragment);
if (scanner.isProjectable()) {
scanner.setTarget(target.toArray());
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
index 5d6a298..7ebcd5c 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
@@ -19,8 +19,6 @@
package org.apache.tajo.storage;
import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -36,7 +34,7 @@ import java.util.Map;
import static org.apache.tajo.conf.TajoConf.ConfVars;
public class StorageManagerFactory {
- private static final Map<String, AbstractStorageManager> storageManagers = Maps.newConcurrentMap();
+ private static final Map<String, AbstractStorageManager> storageManagers = Maps.newHashMap();
public static AbstractStorageManager getStorageManager(TajoConf conf) throws IOException {
return getStorageManager(conf, null);
@@ -83,17 +81,17 @@ public class StorageManagerFactory {
}
public static synchronized SeekableScanner getSeekableScanner(
- TajoConf conf, TableMeta meta, Fragment fragment, Schema schema) throws IOException {
- return (SeekableScanner)getStorageManager(conf, null, false).getScanner(meta, fragment, schema);
+ TajoConf conf, TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException {
+ return (SeekableScanner)getStorageManager(conf, null, false).getScanner(meta, schema, fragment, target);
}
public static synchronized SeekableScanner getSeekableScanner(
- TajoConf conf, TableMeta meta, Path path) throws IOException {
+ TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException {
FileSystem fs = path.getFileSystem(conf);
FileStatus status = fs.getFileStatus(path);
- Fragment fragment = new Fragment(path.getName(), path, meta, 0, status.getLen());
+ Fragment fragment = new Fragment(path.getName(), path, 0, status.getLen());
- return getSeekableScanner(conf, meta, fragment, fragment.getSchema());
+ return getSeekableScanner(conf, meta, schema, fragment, schema);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
index ddfe93d..9627a5d 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -57,8 +57,7 @@ public class StorageUtil {
}
}
- public static void writeTableMeta(Configuration conf, Path tableroot,
- TableMeta meta) throws IOException {
+ public static void writeTableMeta(Configuration conf, Path tableroot, TableMeta meta) throws IOException {
FileSystem fs = tableroot.getFileSystem(conf);
FSDataOutputStream out = fs.create(new Path(tableroot, ".meta"));
FileUtil.writeProto(out, meta.getProto());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
index 151fd1b..41c9d61 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
@@ -19,8 +19,8 @@
package org.apache.tajo.storage;
import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.statistics.ColumnStat;
-import org.apache.tajo.catalog.statistics.TableStat;
+import org.apache.tajo.catalog.statistics.ColumnStats;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.datum.Datum;
@@ -97,16 +97,16 @@ public class TableStatistics {
}
}
- public TableStat getTableStat() {
- TableStat stat = new TableStat();
+ public TableStats getTableStat() {
+ TableStats stat = new TableStats();
- ColumnStat columnStat;
+ ColumnStats columnStats;
for (int i = 0; i < schema.getColumnNum(); i++) {
- columnStat = new ColumnStat(schema.getColumn(i));
- columnStat.setNumNulls(numNulls[i]);
- columnStat.setMinValue(minValues.get(i));
- columnStat.setMaxValue(maxValues.get(i));
- stat.addColumnStat(columnStat);
+ columnStats = new ColumnStats(schema.getColumn(i));
+ columnStats.setNumNulls(numNulls[i]);
+ columnStats.setMinValue(minValues.get(i));
+ columnStats.setMaxValue(maxValues.get(i));
+ stat.addColumnStat(columnStats);
}
stat.setNumRows(this.numRows);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7dc8de28/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java
new file mode 100644
index 0000000..ad19101
--- /dev/null
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/annotation/ForSplitableStore.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface ForSplitableStore {
+}