You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hj...@apache.org on 2014/12/03 06:30:32 UTC
[17/30] tajo git commit: TAJO-1122: Refactor the tajo-storage project
structure.
http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
deleted file mode 100644
index ad5f408..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ /dev/null
@@ -1,857 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.TajoIdProtos;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.datum.NullDatum;
-import org.apache.tajo.datum.ProtobufDatumFactory;
-import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.storage.rcfile.RCFile;
-import org.apache.tajo.storage.sequencefile.SequenceFileScanner;
-import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.KeyValueSet;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-@RunWith(Parameterized.class)
-public class TestStorages {
- private TajoConf conf;
- private static String TEST_PATH = "target/test-data/TestStorages";
-
- private static String TEST_PROJECTION_AVRO_SCHEMA =
- "{\n" +
- " \"type\": \"record\",\n" +
- " \"namespace\": \"org.apache.tajo\",\n" +
- " \"name\": \"testProjection\",\n" +
- " \"fields\": [\n" +
- " { \"name\": \"id\", \"type\": \"int\" },\n" +
- " { \"name\": \"age\", \"type\": \"long\" },\n" +
- " { \"name\": \"score\", \"type\": \"float\" }\n" +
- " ]\n" +
- "}\n";
-
- private static String TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA =
- "{\n" +
- " \"type\": \"record\",\n" +
- " \"namespace\": \"org.apache.tajo\",\n" +
- " \"name\": \"testNullHandlingTypes\",\n" +
- " \"fields\": [\n" +
- " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" +
- " { \"name\": \"col2\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col3\", \"type\": [\"null\", \"string\"] },\n" +
- " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col5\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col6\", \"type\": [\"null\", \"long\"] },\n" +
- " { \"name\": \"col7\", \"type\": [\"null\", \"float\"] },\n" +
- " { \"name\": \"col8\", \"type\": [\"null\", \"double\"] },\n" +
- " { \"name\": \"col9\", \"type\": [\"null\", \"string\"] },\n" +
- " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col11\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col12\", \"type\": \"null\" },\n" +
- " { \"name\": \"col13\", \"type\": [\"null\", \"bytes\"] }\n" +
- " ]\n" +
- "}\n";
-
- private StoreType storeType;
- private boolean splitable;
- private boolean statsable;
- private boolean seekable;
- private Path testDir;
- private FileSystem fs;
-
- public TestStorages(StoreType type, boolean splitable, boolean statsable, boolean seekable) throws IOException {
- this.storeType = type;
- this.splitable = splitable;
- this.statsable = statsable;
- this.seekable = seekable;
-
- conf = new TajoConf();
-
- if (storeType == StoreType.RCFILE) {
- conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100);
- }
-
- testDir = CommonTestingUtil.getTestDir(TEST_PATH);
- fs = testDir.getFileSystem(conf);
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> generateParameters() {
- return Arrays.asList(new Object[][] {
- //type, splitable, statsable, seekable
- {StoreType.CSV, true, true, true},
- {StoreType.RAW, false, true, true},
- {StoreType.RCFILE, true, true, false},
- {StoreType.PARQUET, false, false, false},
- {StoreType.SEQUENCEFILE, true, true, false},
- {StoreType.AVRO, false, false, false},
- {StoreType.TEXTFILE, true, true, false},
- });
- }
-
- @Test
- public void testSplitable() throws IOException {
- if (splitable) {
- Schema schema = new Schema();
- schema.addColumn("id", Type.INT4);
- schema.addColumn("age", Type.INT8);
-
- TableMeta meta = CatalogUtil.newTableMeta(storeType);
- Path tablePath = new Path(testDir, "Splitable.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.enableStats();
- appender.init();
- int tupleNum = 10000;
- VTuple vTuple;
-
- for (int i = 0; i < tupleNum; i++) {
- vTuple = new VTuple(2);
- vTuple.put(0, DatumFactory.createInt4(i + 1));
- vTuple.put(1, DatumFactory.createInt8(25l));
- appender.addTuple(vTuple);
- }
- appender.close();
- TableStats stat = appender.getStats();
- assertEquals(tupleNum, stat.getNumRows().longValue());
-
- FileStatus status = fs.getFileStatus(tablePath);
- long fileLen = status.getLen();
- long randomNum = (long) (Math.random() * fileLen) + 1;
-
- FileFragment[] tablets = new FileFragment[2];
- tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum);
- tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum));
-
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
- assertTrue(scanner.isSplittable());
- scanner.init();
- int tupleCnt = 0;
- while (scanner.next() != null) {
- tupleCnt++;
- }
- scanner.close();
-
- scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[1], schema);
- assertTrue(scanner.isSplittable());
- scanner.init();
- while (scanner.next() != null) {
- tupleCnt++;
- }
- scanner.close();
-
- assertEquals(tupleNum, tupleCnt);
- }
- }
-
- @Test
- public void testRCFileSplitable() throws IOException {
- if (storeType == StoreType.RCFILE) {
- Schema schema = new Schema();
- schema.addColumn("id", Type.INT4);
- schema.addColumn("age", Type.INT8);
-
- TableMeta meta = CatalogUtil.newTableMeta(storeType);
- Path tablePath = new Path(testDir, "Splitable.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.enableStats();
- appender.init();
- int tupleNum = 10000;
- VTuple vTuple;
-
- for (int i = 0; i < tupleNum; i++) {
- vTuple = new VTuple(2);
- vTuple.put(0, DatumFactory.createInt4(i + 1));
- vTuple.put(1, DatumFactory.createInt8(25l));
- appender.addTuple(vTuple);
- }
- appender.close();
- TableStats stat = appender.getStats();
- assertEquals(tupleNum, stat.getNumRows().longValue());
-
- FileStatus status = fs.getFileStatus(tablePath);
- long fileLen = status.getLen();
- long randomNum = 122; // header size
-
- FileFragment[] tablets = new FileFragment[2];
- tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum);
- tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum));
-
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[0], schema);
- assertTrue(scanner.isSplittable());
- scanner.init();
- int tupleCnt = 0;
- while (scanner.next() != null) {
- tupleCnt++;
- }
- scanner.close();
-
- scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, tablets[1], schema);
- assertTrue(scanner.isSplittable());
- scanner.init();
- while (scanner.next() != null) {
- tupleCnt++;
- }
- scanner.close();
-
- assertEquals(tupleNum, tupleCnt);
- }
- }
-
- @Test
- public void testProjection() throws IOException {
- Schema schema = new Schema();
- schema.addColumn("id", Type.INT4);
- schema.addColumn("age", Type.INT8);
- schema.addColumn("score", Type.FLOAT4);
-
- TableMeta meta = CatalogUtil.newTableMeta(storeType);
- meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
- if (storeType == StoreType.AVRO) {
- meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
- TEST_PROJECTION_AVRO_SCHEMA);
- }
-
- Path tablePath = new Path(testDir, "testProjection.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.init();
- int tupleNum = 10000;
- VTuple vTuple;
-
- for (int i = 0; i < tupleNum; i++) {
- vTuple = new VTuple(3);
- vTuple.put(0, DatumFactory.createInt4(i + 1));
- vTuple.put(1, DatumFactory.createInt8(i + 2));
- vTuple.put(2, DatumFactory.createFloat4(i + 3));
- appender.addTuple(vTuple);
- }
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen());
-
- Schema target = new Schema();
- target.addColumn("age", Type.INT8);
- target.addColumn("score", Type.FLOAT4);
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment, target);
- scanner.init();
- int tupleCnt = 0;
- Tuple tuple;
- while ((tuple = scanner.next()) != null) {
- if (storeType == StoreType.RCFILE
- || storeType == StoreType.CSV
- || storeType == StoreType.PARQUET
- || storeType == StoreType.SEQUENCEFILE
- || storeType == StoreType.AVRO) {
- assertTrue(tuple.get(0) == null);
- }
- assertTrue(tupleCnt + 2 == tuple.get(1).asInt8());
- assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4());
- tupleCnt++;
- }
- scanner.close();
-
- assertEquals(tupleNum, tupleCnt);
- }
-
- @Test
- public void testVariousTypes() throws IOException {
- Schema schema = new Schema();
- schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
-
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
- if (storeType == StoreType.AVRO) {
- String path = FileUtil.getResourcePath("testVariousTypes.avsc").toString();
- meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
- }
-
- Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.init();
-
- QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
-
- Tuple tuple = new VTuple(13);
- tuple.put(new Datum[] {
- DatumFactory.createBool(true),
- DatumFactory.createBit((byte) 0x99),
- DatumFactory.createChar("hyunsik"),
- DatumFactory.createInt2((short) 17),
- DatumFactory.createInt4(59),
- DatumFactory.createInt8(23l),
- DatumFactory.createFloat4(77.9f),
- DatumFactory.createFloat8(271.9f),
- DatumFactory.createText("hyunsik"),
- DatumFactory.createBlob("hyunsik".getBytes()),
- DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
- factory.createDatum(queryid.getProto())
- });
- appender.addTuple(tuple);
- appender.flush();
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
- scanner.init();
-
- Tuple retrieved;
- while ((retrieved = scanner.next()) != null) {
- for (int i = 0; i < tuple.size(); i++) {
- assertEquals(tuple.get(i), retrieved.get(i));
- }
- }
- scanner.close();
- }
-
- @Test
- public void testNullHandlingTypes() throws IOException {
- Schema schema = new Schema();
- schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
-
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
- meta.putOption(StorageConstants.TEXT_NULL, "\\\\N");
- meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N");
- meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName());
- meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\");
- if (storeType == StoreType.AVRO) {
- meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL,
- TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA);
- }
-
- Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.init();
-
- QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
-
- Tuple seedTuple = new VTuple(13);
- seedTuple.put(new Datum[]{
- DatumFactory.createBool(true), // 0
- DatumFactory.createBit((byte) 0x99), // 1
- DatumFactory.createChar("hyunsik"), // 2
- DatumFactory.createInt2((short) 17), // 3
- DatumFactory.createInt4(59), // 4
- DatumFactory.createInt8(23l), // 5
- DatumFactory.createFloat4(77.9f), // 6
- DatumFactory.createFloat8(271.9f), // 7
- DatumFactory.createText("hyunsik"), // 8
- DatumFactory.createBlob("hyunsik".getBytes()),// 9
- DatumFactory.createInet4("192.168.0.1"), // 10
- NullDatum.get(), // 11
- factory.createDatum(queryid.getProto()) // 12
- });
-
- // Making tuples with different null column positions
- Tuple tuple;
- for (int i = 0; i < 13; i++) {
- tuple = new VTuple(13);
- for (int j = 0; j < 13; j++) {
- if (i == j) { // i'th column will have NULL value
- tuple.put(j, NullDatum.get());
- } else {
- tuple.put(j, seedTuple.get(j));
- }
- }
- appender.addTuple(tuple);
- }
- appender.flush();
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
- scanner.init();
-
- Tuple retrieved;
- int i = 0;
- while ((retrieved = scanner.next()) != null) {
- assertEquals(13, retrieved.size());
- for (int j = 0; j < 13; j++) {
- if (i == j) {
- assertEquals(NullDatum.get(), retrieved.get(j));
- } else {
- assertEquals(seedTuple.get(j), retrieved.get(j));
- }
- }
-
- i++;
- }
- scanner.close();
- }
-
- @Test
- public void testRCFileTextSerializeDeserialize() throws IOException {
- if(storeType != StoreType.RCFILE) return;
-
- Schema schema = new Schema();
- schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
-
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
-
- Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.enableStats();
- appender.init();
-
- QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
-
- Tuple tuple = new VTuple(13);
- tuple.put(new Datum[] {
- DatumFactory.createBool(true),
- DatumFactory.createBit((byte) 0x99),
- DatumFactory.createChar("jinho"),
- DatumFactory.createInt2((short) 17),
- DatumFactory.createInt4(59),
- DatumFactory.createInt8(23l),
- DatumFactory.createFloat4(77.9f),
- DatumFactory.createFloat8(271.9f),
- DatumFactory.createText("jinho"),
- DatumFactory.createBlob("hyunsik babo".getBytes()),
- DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
- factory.createDatum(queryid.getProto())
- });
- appender.addTuple(tuple);
- appender.flush();
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
-
- FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
- scanner.init();
-
- Tuple retrieved;
- while ((retrieved=scanner.next()) != null) {
- for (int i = 0; i < tuple.size(); i++) {
- assertEquals(tuple.get(i), retrieved.get(i));
- }
- }
- scanner.close();
- assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
- assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
- }
-
- @Test
- public void testRCFileBinarySerializeDeserialize() throws IOException {
- if(storeType != StoreType.RCFILE) return;
-
- Schema schema = new Schema();
- schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
-
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
-
- Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.enableStats();
- appender.init();
-
- QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
-
- Tuple tuple = new VTuple(13);
- tuple.put(new Datum[] {
- DatumFactory.createBool(true),
- DatumFactory.createBit((byte) 0x99),
- DatumFactory.createChar("jinho"),
- DatumFactory.createInt2((short) 17),
- DatumFactory.createInt4(59),
- DatumFactory.createInt8(23l),
- DatumFactory.createFloat4(77.9f),
- DatumFactory.createFloat8(271.9f),
- DatumFactory.createText("jinho"),
- DatumFactory.createBlob("hyunsik babo".getBytes()),
- DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
- factory.createDatum(queryid.getProto())
- });
- appender.addTuple(tuple);
- appender.flush();
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
-
- FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
- scanner.init();
-
- Tuple retrieved;
- while ((retrieved=scanner.next()) != null) {
- for (int i = 0; i < tuple.size(); i++) {
- assertEquals(tuple.get(i), retrieved.get(i));
- }
- }
- scanner.close();
- assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
- assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
- }
-
- @Test
- public void testSequenceFileTextSerializeDeserialize() throws IOException {
- if(storeType != StoreType.SEQUENCEFILE) return;
-
- Schema schema = new Schema();
- schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
-
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
-
- Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.enableStats();
- appender.init();
-
- QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
-
- Tuple tuple = new VTuple(13);
- tuple.put(new Datum[] {
- DatumFactory.createBool(true),
- DatumFactory.createBit((byte) 0x99),
- DatumFactory.createChar("jinho"),
- DatumFactory.createInt2((short) 17),
- DatumFactory.createInt4(59),
- DatumFactory.createInt8(23l),
- DatumFactory.createFloat4(77.9f),
- DatumFactory.createFloat8(271.9f),
- DatumFactory.createText("jinho"),
- DatumFactory.createBlob("hyunsik babo".getBytes()),
- DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
- factory.createDatum(queryid.getProto())
- });
- appender.addTuple(tuple);
- appender.flush();
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
-
- FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
- scanner.init();
-
- assertTrue(scanner instanceof SequenceFileScanner);
- Writable key = ((SequenceFileScanner) scanner).getKey();
- assertEquals(key.getClass().getCanonicalName(), LongWritable.class.getCanonicalName());
-
- Tuple retrieved;
- while ((retrieved=scanner.next()) != null) {
- for (int i = 0; i < tuple.size(); i++) {
- assertEquals(tuple.get(i), retrieved.get(i));
- }
- }
- scanner.close();
- assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
- assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
- }
-
- @Test
- public void testSequenceFileBinarySerializeDeserialize() throws IOException {
- if(storeType != StoreType.SEQUENCEFILE) return;
-
- Schema schema = new Schema();
- schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
-
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName());
-
- Path tablePath = new Path(testDir, "testVariousTypes.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.enableStats();
- appender.init();
-
- QueryId queryid = new QueryId("12345", 5);
- ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
-
- Tuple tuple = new VTuple(13);
- tuple.put(new Datum[] {
- DatumFactory.createBool(true),
- DatumFactory.createBit((byte) 0x99),
- DatumFactory.createChar("jinho"),
- DatumFactory.createInt2((short) 17),
- DatumFactory.createInt4(59),
- DatumFactory.createInt8(23l),
- DatumFactory.createFloat4(77.9f),
- DatumFactory.createFloat8(271.9f),
- DatumFactory.createText("jinho"),
- DatumFactory.createBlob("hyunsik babo".getBytes()),
- DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
- factory.createDatum(queryid.getProto())
- });
- appender.addTuple(tuple);
- appender.flush();
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
-
- FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
- scanner.init();
-
- assertTrue(scanner instanceof SequenceFileScanner);
- Writable key = ((SequenceFileScanner) scanner).getKey();
- assertEquals(key.getClass().getCanonicalName(), BytesWritable.class.getCanonicalName());
-
- Tuple retrieved;
- while ((retrieved=scanner.next()) != null) {
- for (int i = 0; i < tuple.size(); i++) {
- assertEquals(tuple.get(i), retrieved.get(i));
- }
- }
- scanner.close();
- assertEquals(appender.getStats().getNumBytes().longValue(), scanner.getInputStats().getNumBytes().longValue());
- assertEquals(appender.getStats().getNumRows().longValue(), scanner.getInputStats().getNumRows().longValue());
- }
-
- @Test
- public void testTime() throws IOException {
- if (storeType == StoreType.CSV || storeType == StoreType.RAW) {
- Schema schema = new Schema();
- schema.addColumn("col1", Type.DATE);
- schema.addColumn("col2", Type.TIME);
- schema.addColumn("col3", Type.TIMESTAMP);
-
- KeyValueSet options = new KeyValueSet();
- TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
-
- Path tablePath = new Path(testDir, "testTime.data");
- Appender appender = StorageManager.getFileStorageManager(conf).getAppender(meta, schema, tablePath);
- appender.init();
-
- Tuple tuple = new VTuple(3);
- tuple.put(new Datum[]{
- DatumFactory.createDate("1980-04-01"),
- DatumFactory.createTime("12:34:56"),
- DatumFactory.createTimestmpDatumWithUnixTime((int)(System.currentTimeMillis() / 1000))
- });
- appender.addTuple(tuple);
- appender.flush();
- appender.close();
-
- FileStatus status = fs.getFileStatus(tablePath);
- FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema, fragment);
- scanner.init();
-
- Tuple retrieved;
- while ((retrieved = scanner.next()) != null) {
- for (int i = 0; i < tuple.size(); i++) {
- assertEquals(tuple.get(i), retrieved.get(i));
- }
- }
- scanner.close();
- }
- }
-
- @Test
- public void testSeekableScanner() throws IOException {
- if (!seekable) {
- return;
- }
-
- Schema schema = new Schema();
- schema.addColumn("id", Type.INT4);
- schema.addColumn("age", Type.INT8);
- schema.addColumn("comment", Type.TEXT);
-
- TableMeta meta = CatalogUtil.newTableMeta(storeType);
- Path tablePath = new Path(testDir, "Seekable.data");
- FileAppender appender = (FileAppender) StorageManager.getFileStorageManager(conf).getAppender(meta, schema,
- tablePath);
- appender.enableStats();
- appender.init();
- int tupleNum = 100000;
- VTuple vTuple;
-
- List<Long> offsets = Lists.newArrayList();
- offsets.add(0L);
- for (int i = 0; i < tupleNum; i++) {
- vTuple = new VTuple(3);
- vTuple.put(0, DatumFactory.createInt4(i + 1));
- vTuple.put(1, DatumFactory.createInt8(25l));
- vTuple.put(2, DatumFactory.createText("test" + i));
- appender.addTuple(vTuple);
-
- // find a seek position
- if (i % (tupleNum / 3) == 0) {
- offsets.add(appender.getOffset());
- }
- }
-
- // end of file
- if (!offsets.contains(appender.getOffset())) {
- offsets.add(appender.getOffset());
- }
-
- appender.close();
- if (statsable) {
- TableStats stat = appender.getStats();
- assertEquals(tupleNum, stat.getNumRows().longValue());
- }
-
- FileStatus status = fs.getFileStatus(tablePath);
- assertEquals(status.getLen(), appender.getOffset());
-
- Scanner scanner;
- int tupleCnt = 0;
- long prevOffset = 0;
- long readBytes = 0;
- long readRows = 0;
- for (long offset : offsets) {
- scanner = StorageManager.getFileStorageManager(conf).getScanner(meta, schema,
- new FileFragment("table", tablePath, prevOffset, offset - prevOffset), schema);
- scanner.init();
-
- while (scanner.next() != null) {
- tupleCnt++;
- }
-
- scanner.close();
- if (statsable) {
- readBytes += scanner.getInputStats().getNumBytes();
- readRows += scanner.getInputStats().getNumRows();
- }
- prevOffset = offset;
- }
-
- assertEquals(tupleNum, tupleCnt);
- if (statsable) {
- assertEquals(appender.getStats().getNumBytes().longValue(), readBytes);
- assertEquals(appender.getStats().getNumRows().longValue(), readRows);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java
deleted file mode 100644
index 639ca04..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestTupleComparator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage;
-
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.DatumFactory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestTupleComparator {
-
- @Before
- public void setUp() throws Exception {
- }
-
- @After
- public void tearDown() throws Exception {
- }
-
- @Test
- public final void testCompare() {
- Schema schema = new Schema();
- schema.addColumn("col1", Type.INT4);
- schema.addColumn("col2", Type.INT4);
- schema.addColumn("col3", Type.INT4);
- schema.addColumn("col4", Type.INT4);
- schema.addColumn("col5", Type.TEXT);
-
- Tuple tuple1 = new VTuple(5);
- Tuple tuple2 = new VTuple(5);
-
- tuple1.put(
- new Datum[] {
- DatumFactory.createInt4(9),
- DatumFactory.createInt4(3),
- DatumFactory.createInt4(33),
- DatumFactory.createInt4(4),
- DatumFactory.createText("abc")});
- tuple2.put(
- new Datum[] {
- DatumFactory.createInt4(1),
- DatumFactory.createInt4(25),
- DatumFactory.createInt4(109),
- DatumFactory.createInt4(4),
- DatumFactory.createText("abd")});
-
- SortSpec sortKey1 = new SortSpec(schema.getColumn("col4"), true, false);
- SortSpec sortKey2 = new SortSpec(schema.getColumn("col5"), true, false);
-
- BaseTupleComparator tc = new BaseTupleComparator(schema,
- new SortSpec[] {sortKey1, sortKey2});
- assertEquals(-1, tc.compare(tuple1, tuple2));
- assertEquals(1, tc.compare(tuple2, tuple1));
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java
deleted file mode 100644
index 9837fd1..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestVTuple.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage;
-
-
-import org.junit.Before;
-import org.junit.Test;
-import org.apache.tajo.datum.DatumFactory;
-
-import static org.junit.Assert.*;
-
-public class TestVTuple {
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
-
- }
-
- @Test
- public void testContain() {
- VTuple t1 = new VTuple(260);
- t1.put(0, DatumFactory.createInt4(1));
- t1.put(1, DatumFactory.createInt4(1));
- t1.put(27, DatumFactory.createInt4(1));
- t1.put(96, DatumFactory.createInt4(1));
- t1.put(257, DatumFactory.createInt4(1));
-
- assertTrue(t1.contains(0));
- assertTrue(t1.contains(1));
- assertFalse(t1.contains(2));
- assertFalse(t1.contains(3));
- assertFalse(t1.contains(4));
- assertTrue(t1.contains(27));
- assertFalse(t1.contains(28));
- assertFalse(t1.contains(95));
- assertTrue(t1.contains(96));
- assertFalse(t1.contains(97));
- assertTrue(t1.contains(257));
- }
-
- @Test
- public void testPut() {
- VTuple t1 = new VTuple(260);
- t1.put(0, DatumFactory.createText("str"));
- t1.put(1, DatumFactory.createInt4(2));
- t1.put(257, DatumFactory.createFloat4(0.76f));
-
- assertTrue(t1.contains(0));
- assertTrue(t1.contains(1));
-
- assertEquals(t1.getText(0),"str");
- assertEquals(t1.get(1).asInt4(),2);
- assertTrue(t1.get(257).asFloat4() == 0.76f);
- }
-
- @Test
- public void testEquals() {
- Tuple t1 = new VTuple(5);
- Tuple t2 = new VTuple(5);
-
- t1.put(0, DatumFactory.createInt4(1));
- t1.put(1, DatumFactory.createInt4(2));
- t1.put(3, DatumFactory.createInt4(2));
-
- t2.put(0, DatumFactory.createInt4(1));
- t2.put(1, DatumFactory.createInt4(2));
- t2.put(3, DatumFactory.createInt4(2));
-
- assertEquals(t1,t2);
-
- Tuple t3 = new VTuple(5);
- t2.put(0, DatumFactory.createInt4(1));
- t2.put(1, DatumFactory.createInt4(2));
- t2.put(4, DatumFactory.createInt4(2));
-
- assertNotSame(t1,t3);
- }
-
- @Test
- public void testHashCode() {
- Tuple t1 = new VTuple(5);
- Tuple t2 = new VTuple(5);
-
- t1.put(0, DatumFactory.createInt4(1));
- t1.put(1, DatumFactory.createInt4(2));
- t1.put(3, DatumFactory.createInt4(2));
- t1.put(4, DatumFactory.createText("hyunsik"));
-
- t2.put(0, DatumFactory.createInt4(1));
- t2.put(1, DatumFactory.createInt4(2));
- t2.put(3, DatumFactory.createInt4(2));
- t2.put(4, DatumFactory.createText("hyunsik"));
-
- assertEquals(t1.hashCode(),t2.hashCode());
-
- Tuple t3 = new VTuple(5);
- t3.put(0, DatumFactory.createInt4(1));
- t3.put(1, DatumFactory.createInt4(2));
- t3.put(4, DatumFactory.createInt4(2));
-
- assertNotSame(t1.hashCode(),t3.hashCode());
- }
-
- @Test
- public void testPutTuple() {
- Tuple t1 = new VTuple(5);
-
- t1.put(0, DatumFactory.createInt4(1));
- t1.put(1, DatumFactory.createInt4(2));
- t1.put(2, DatumFactory.createInt4(3));
-
- Tuple t2 = new VTuple(2);
- t2.put(0, DatumFactory.createInt4(4));
- t2.put(1, DatumFactory.createInt4(5));
-
- t1.put(3, t2);
-
- for (int i = 0; i < 5; i++) {
- assertEquals(i+1, t1.get(i).asInt4());
- }
- }
-
- @Test
- public void testClone() throws CloneNotSupportedException {
- Tuple t1 = new VTuple(5);
-
- t1.put(0, DatumFactory.createInt4(1));
- t1.put(1, DatumFactory.createInt4(2));
- t1.put(3, DatumFactory.createInt4(2));
- t1.put(4, DatumFactory.createText("str"));
-
- VTuple t2 = (VTuple) t1.clone();
- assertNotSame(t1, t2);
- assertEquals(t1, t2);
-
- assertSame(t1.get(4), t2.get(4));
-
- t1.clear();
- assertFalse(t1.equals(t2));
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
deleted file mode 100644
index 6186e9e..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.avro;
-
-import org.apache.avro.Schema;
-import org.apache.tajo.HttpFileServer;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.storage.StorageConstants;
-import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.NetUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests for {@link org.apache.tajo.storage.avro.AvroUtil}.
- */
-public class TestAvroUtil {
- private Schema expected;
- private URL schemaUrl;
-
- @Before
- public void setUp() throws Exception {
- schemaUrl = FileUtil.getResourcePath("testVariousTypes.avsc");
- assertNotNull(schemaUrl);
-
- File file = new File(schemaUrl.getPath());
- assertTrue(file.exists());
-
- expected = new Schema.Parser().parse(file);
- }
-
- @Test
- public void testGetSchema() throws IOException, URISyntaxException {
- TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
- meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, FileUtil.readTextFile(new File(schemaUrl.getPath())));
- Schema schema = AvroUtil.getAvroSchema(meta, new TajoConf());
- assertEquals(expected, schema);
-
- meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
- meta.putOption(StorageConstants.AVRO_SCHEMA_URL, schemaUrl.getPath());
- schema = AvroUtil.getAvroSchema(meta, new TajoConf());
- assertEquals(expected, schema);
-
- HttpFileServer server = new HttpFileServer(NetUtils.createSocketAddr("127.0.0.1:0"));
- try {
- server.start();
- InetSocketAddress addr = server.getBindAddress();
-
- String url = "http://127.0.0.1:" + addr.getPort() + schemaUrl.getPath();
- meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.AVRO);
- meta.putOption(StorageConstants.AVRO_SCHEMA_URL, url);
- schema = AvroUtil.getAvroSchema(meta, new TajoConf());
- } finally {
- server.stop();
- }
- assertEquals(expected, schema);
- }
-
- @Test
- public void testGetSchemaFromHttp() throws IOException, URISyntaxException {
- HttpFileServer server = new HttpFileServer(NetUtils.createSocketAddr("127.0.0.1:0"));
- try {
- server.start();
- InetSocketAddress addr = server.getBindAddress();
-
- Schema schema = AvroUtil.getAvroSchemaFromHttp("http://127.0.0.1:" + addr.getPort() + schemaUrl.getPath());
- assertEquals(expected, schema);
- } finally {
- server.stop();
- }
- }
-
- @Test
- public void testGetSchemaFromFileSystem() throws IOException, URISyntaxException {
- Schema schema = AvroUtil.getAvroSchemaFromFileSystem(schemaUrl.toString(), new TajoConf());
-
- assertEquals(expected, schema);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java
deleted file mode 100644
index 39d28b3..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.hbase;
-
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.util.KeyValueSet;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class TestColumnMapping {
- @Test
- public void testColumnKeyValueMapping() throws Exception {
- KeyValueSet keyValueSet = new KeyValueSet();
- keyValueSet.set(HBaseStorageConstants.META_TABLE_KEY, "test");
- keyValueSet.set(HBaseStorageConstants.META_COLUMNS_KEY, ":key,col2:key:,col2:value:#b,col3:");
-
- Schema schema = new Schema();
- schema.addColumn("c1", Type.TEXT);
- schema.addColumn("c2", Type.TEXT);
- schema.addColumn("c3", Type.TEXT);
- schema.addColumn("c4", Type.TEXT);
-
- TableMeta tableMeta = new TableMeta(StoreType.HBASE, keyValueSet);
-
- ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta);
-
- List<String> cfNames = columnMapping.getColumnFamilyNames();
- assertEquals(2, cfNames.size());
- assertEquals("col2", cfNames.get(0));
- assertEquals("col3", cfNames.get(1));
-
- for (int i = 0; i < columnMapping.getIsBinaryColumns().length; i++) {
- if (i == 2) {
- assertTrue(columnMapping.getIsBinaryColumns()[i]);
- } else {
- assertFalse(columnMapping.getIsBinaryColumns()[i]);
- }
- }
-
- for (int i = 0; i < columnMapping.getIsRowKeyMappings().length; i++) {
- if (i == 0) {
- assertTrue(columnMapping.getIsRowKeyMappings()[i]);
- } else {
- assertFalse(columnMapping.getIsRowKeyMappings()[i]);
- }
- }
-
- String[] expectedColumnNames = { null, null, null, null};
- for (int i = 0; i < schema.size(); i++) {
- String columnName = columnMapping.getMappingColumns()[i][1] == null ? null :
- new String(columnMapping.getMappingColumns()[i][1]);
- assertEquals(expectedColumnNames[i], columnName);
- }
-
- for (int i = 0; i < schema.size(); i++) {
- if (i == 1) {
- assertTrue(columnMapping.getIsColumnKeys()[i]);
- } else {
- assertFalse(columnMapping.getIsColumnKeys()[i]);
- }
- }
-
- for (int i = 0; i < schema.size(); i++) {
- if (i == 2) {
- assertTrue(columnMapping.getIsColumnValues()[i]);
- } else {
- assertFalse(columnMapping.getIsColumnValues()[i]);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/dfd7f996/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java
deleted file mode 100644
index 1fc4065..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/hbase/TestHBaseStorageManager.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.hbase;
-
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.TextDatum;
-import org.apache.tajo.plan.expr.*;
-import org.apache.tajo.plan.logical.ScanNode;
-import org.apache.tajo.storage.StorageManager;
-import org.apache.tajo.util.Pair;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-public class TestHBaseStorageManager {
- @Test
- public void testGetIndexPredications() throws Exception {
- Column rowkeyColumn = new Column("rk", Type.TEXT);
- // where rk >= '020' and rk <= '055'
- ScanNode scanNode = new ScanNode(1);
- EvalNode evalNode1 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("020")));
- EvalNode evalNode2 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("055")));
- EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2);
- scanNode.setQual(evalNodeA);
-
- HBaseStorageManager storageManager =
- (HBaseStorageManager) StorageManager.getStorageManager(new TajoConf(), StoreType.HBASE);
- List<Set<EvalNode>> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
- assertNotNull(indexEvals);
- assertEquals(1, indexEvals.size());
- Pair<Datum, Datum> indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
- assertEquals("020", indexPredicateValue.getFirst().asChars());
- assertEquals("055", indexPredicateValue.getSecond().asChars());
-
- // where (rk >= '020' and rk <= '055') or rk = '075'
- EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(rowkeyColumn),new ConstEval(new TextDatum("075")));
- EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3);
- scanNode.setQual(evalNodeB);
- indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
- assertEquals(2, indexEvals.size());
- indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
- assertEquals("020", indexPredicateValue.getFirst().asChars());
- assertEquals("055", indexPredicateValue.getSecond().asChars());
-
- indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1));
- assertEquals("075", indexPredicateValue.getFirst().asChars());
- assertEquals("075", indexPredicateValue.getSecond().asChars());
-
- // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078')
- EvalNode evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("072")));
- EvalNode evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("078")));
- EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5);
- EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC);
- scanNode.setQual(evalNodeD);
- indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
- assertEquals(2, indexEvals.size());
-
- indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
- assertEquals("020", indexPredicateValue.getFirst().asChars());
- assertEquals("055", indexPredicateValue.getSecond().asChars());
-
- indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1));
- assertEquals("072", indexPredicateValue.getFirst().asChars());
- assertEquals("078", indexPredicateValue.getSecond().asChars());
-
- // where (rk >= '020' and rk <= '055') or (rk >= '072' and rk <= '078' and rk >= '073')
- evalNode4 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("072")));
- evalNode5 = new BinaryEval(EvalType.LEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("078")));
- evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5);
- EvalNode evalNode6 = new BinaryEval(EvalType.GEQ, new FieldEval(rowkeyColumn), new ConstEval(new TextDatum("073")));
- evalNodeD = new BinaryEval(EvalType.AND, evalNodeC, evalNode6);
- EvalNode evalNodeE = new BinaryEval(EvalType.OR, evalNodeA, evalNodeD);
- scanNode.setQual(evalNodeE);
- indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
- assertEquals(2, indexEvals.size());
-
- indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
- assertEquals("020", indexPredicateValue.getFirst().asChars());
- assertEquals("055", indexPredicateValue.getSecond().asChars());
-
- indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(1));
- assertEquals("073", indexPredicateValue.getFirst().asChars());
- assertEquals("078", indexPredicateValue.getSecond().asChars());
- }
-}