You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2015/09/17 14:42:24 UTC
[1/4] tajo git commit: TAJO-1730: JDBC Tablespace support.
Repository: tajo
Updated Branches:
refs/heads/master e1c2d352e -> 0e4ad5635
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLEndPointTests.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLEndPointTests.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLEndPointTests.java
new file mode 100644
index 0000000..111f52b
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLEndPointTests.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import com.google.common.collect.Sets;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.client.TajoClient;
+import org.apache.tajo.exception.UndefinedTableException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestPgSQLEndPointTests extends QueryTestCaseBase {
+ private static final String jdbcUrl = PgSQLTestServer.getInstance().getJdbcUrlForAdmin();
+ private static TajoClient client;
+
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ QueryTestCaseBase.testingCluster.getMaster().refresh();
+ client = QueryTestCaseBase.testingCluster.newTajoClient();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ client.close();
+ }
+
+ @Test(timeout = 1000)
+ public void testGetAllDatabaseNames() {
+ Set<String> retrieved = Sets.newHashSet(client.getAllDatabaseNames());
+ assertTrue(retrieved.contains(PgSQLTestServer.DATABASE_NAME));
+ }
+
+ @Test(timeout = 1000)
+ public void testGetTableList() {
+ final Set<String> expected = Sets.newHashSet(PgSQLTestServer.TPCH_TABLES);
+ expected.add("datetime_types");
+ final Set<String> retrieved = Sets.newHashSet(client.getTableList("tpch"));
+
+ assertEquals(expected, retrieved);
+ }
+
+ @Test(timeout = 1000)
+ public void testGetTable() throws UndefinedTableException {
+ for (String tableName: PgSQLTestServer.TPCH_TABLES) {
+ TableDesc retrieved = client.getTableDesc(PgSQLTestServer.DATABASE_NAME + "." + tableName);
+ assertEquals(PgSQLTestServer.DATABASE_NAME + "." + tableName, retrieved.getName());
+ assertEquals(jdbcUrl + "&table=" + tableName, retrieved.getUri().toASCIIString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java
new file mode 100644
index 0000000..cf4fe12
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLJdbcTableSpace.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import net.minidev.json.JSONObject;
+import org.apache.tajo.catalog.MetadataProvider;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.exception.TajoRuntimeException;
+import org.apache.tajo.storage.Tablespace;
+import org.apache.tajo.storage.TablespaceManager;
+import org.apache.tajo.storage.fragment.Fragment;
+import org.apache.tajo.storage.jdbc.JdbcTablespace;
+import org.junit.Test;
+import org.postgresql.util.PSQLException;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+public class TestPgSQLJdbcTableSpace {
+ static String jdbcUrl = PgSQLTestServer.getInstance().getJdbcUrlForAdmin();
+
+ @Test(timeout = 1000)
+ public void testTablespaceHandler() throws Exception {
+ assertTrue((TablespaceManager.getByName("pgsql_cluster").get()) instanceof PgSQLTablespace);
+ assertEquals("pgsql_cluster", (TablespaceManager.getByName("pgsql_cluster").get().getName()));
+
+ assertTrue((TablespaceManager.get(jdbcUrl).get()) instanceof PgSQLTablespace);
+ assertTrue((TablespaceManager.get(jdbcUrl + "&table=tb1").get()) instanceof PgSQLTablespace);
+
+ assertEquals(jdbcUrl, TablespaceManager.get(jdbcUrl).get().getUri().toASCIIString());
+ assertTrue(TablespaceManager.get(jdbcUrl).get().getMetadataProvider() instanceof PgSQLMetadataProvider);
+ }
+
+ @Test(timeout = 1000, expected = TajoRuntimeException.class)
+ public void testCreateTable() throws IOException, TajoException {
+ Tablespace space = TablespaceManager.getByName("pgsql_cluster").get();
+ space.createTable(null, false);
+ }
+
+ @Test(timeout = 1000, expected = TajoRuntimeException.class)
+ public void testDropTable() throws IOException, TajoException {
+ Tablespace space = TablespaceManager.getByName("pgsql_cluster").get();
+ space.purgeTable(null);
+ }
+
+ @Test(timeout = 1000)
+ public void testGetSplits() throws IOException, TajoException {
+ Tablespace space = TablespaceManager.getByName("pgsql_cluster").get();
+ MetadataProvider provider = space.getMetadataProvider();
+ TableDesc table = provider.getTableDesc(null, "lineitem");
+ List<Fragment> fragments = space.getSplits("lineitem", table, null);
+ assertNotNull(fragments);
+ assertEquals(1, fragments.size());
+ }
+
+ @Test
+ public void testConnProperties() throws Exception {
+ Map<String, String> connProperties = new HashMap<>();
+ connProperties.put("user", "postgres");
+ connProperties.put("password", "");
+
+ String uri = PgSQLTestServer.getInstance().getJdbcUrl().split("\\?")[0];
+ Tablespace space = new PgSQLTablespace("t1", URI.create(uri), getJsonTablespace(connProperties));
+ try {
+ space.init(new TajoConf());
+ } finally {
+ space.close();
+ }
+ }
+
+ @Test
+ public void testConnPropertiesNegative() throws Exception {
+ Map<String, String> connProperties = new HashMap<>();
+ connProperties.put("user", "postgresX");
+ connProperties.put("password", "");
+
+ String uri = PgSQLTestServer.getInstance().getJdbcUrl().split("\\?")[0];
+ Tablespace space = new PgSQLTablespace("t1", URI.create(uri), getJsonTablespace(connProperties));
+ try {
+ space.init(new TajoConf());
+ fail("Must be failed");
+ } catch (IOException ioe) {
+ assertTrue(ioe.getCause() instanceof PSQLException);
+ } finally {
+ space.close();
+ }
+ }
+
+ public static JSONObject getJsonTablespace(Map<String, String> connProperties)
+ throws IOException {
+ String uri = PgSQLTestServer.getInstance().getJdbcUrl().split("\\?")[0];
+
+ JSONObject configElements = new JSONObject();
+ configElements.put(JdbcTablespace.CONFIG_KEY_MAPPED_DATABASE, PgSQLTestServer.DATABASE_NAME);
+
+ JSONObject connPropertiesJson = new JSONObject();
+ for (Map.Entry<String, String> entry : connProperties.entrySet()) {
+ connPropertiesJson.put(entry.getKey(), entry.getValue());
+ }
+ configElements.put(JdbcTablespace.CONFIG_KEY_CONN_PROPERTIES, connPropertiesJson);
+
+ return configElements;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java
new file mode 100644
index 0000000..2221f94
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLMetadataProvider.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import com.google.common.collect.Sets;
+import org.apache.tajo.catalog.MetadataProvider;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.storage.Tablespace;
+import org.apache.tajo.storage.TablespaceManager;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.Set;
+
+import static org.junit.Assert.*;
+
+public class TestPgSQLMetadataProvider {
+ static final String jdbcUrl = PgSQLTestServer.getInstance().getJdbcUrlForAdmin();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ }
+
+ @Test
+ public void testGetTablespaceName() throws Exception {
+ Tablespace tablespace = TablespaceManager.get(jdbcUrl).get();
+ MetadataProvider provider = tablespace.getMetadataProvider();
+ assertEquals("pgsql_cluster", provider.getTablespaceName());
+ }
+
+ @Test
+ public void testGetDatabaseName() throws Exception {
+ Tablespace tablespace = TablespaceManager.get(jdbcUrl).get();
+ MetadataProvider provider = tablespace.getMetadataProvider();
+ assertEquals("tpch", provider.getDatabaseName());
+ }
+
+ @Test
+ public void testGetSchemas() throws Exception {
+ Tablespace tablespace = TablespaceManager.get(jdbcUrl).get();
+ MetadataProvider provider = tablespace.getMetadataProvider();
+ assertTrue(provider.getSchemas().isEmpty());
+ }
+
+ @Test
+ public void testGetTables() throws Exception {
+ Tablespace tablespace = TablespaceManager.get(jdbcUrl).get();
+ MetadataProvider provider = tablespace.getMetadataProvider();
+
+ final Set<String> expected = Sets.newHashSet(PgSQLTestServer.TPCH_TABLES);
+ expected.add("datetime_types");
+ final Set<String> found = Sets.newHashSet(provider.getTables(null, null));
+
+ assertEquals(expected, found);
+ }
+
+ @Test
+ public void testGetTableDescriptor() throws Exception {
+ Tablespace tablespace = TablespaceManager.get(jdbcUrl).get();
+ MetadataProvider provider = tablespace.getMetadataProvider();
+
+ for (String tableName : PgSQLTestServer.TPCH_TABLES) {
+ TableDesc table = provider.getTableDesc(null, tableName);
+ assertEquals("tpch." + tableName, table.getName());
+ assertEquals(jdbcUrl + "&table=" + tableName, table.getUri().toASCIIString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java
new file mode 100644
index 0000000..9655968
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLQueryTests.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import com.google.common.base.Optional;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.Tablespace;
+import org.apache.tajo.storage.TablespaceManager;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestPgSQLQueryTests extends QueryTestCaseBase {
+ @SuppressWarnings("unused")
+ // This should be invoked for initializing PgSQLTestServer
+ private static final String jdbcUrl = PgSQLTestServer.getInstance().getJdbcUrl();
+
+ public TestPgSQLQueryTests() {
+ super(PgSQLTestServer.DATABASE_NAME);
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ QueryTestCaseBase.testingCluster.getMaster().refresh();
+ }
+
+ @SimpleTest
+ @Test
+ public void testProjectedColumns() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testFixedLengthFields() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testVariableLengthFields() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testDateTimeTypes() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testTPCH_Q1() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testTPCH_Q2_Part() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testTPCH_Q2_Part_MixedStorage() throws Exception {
+ // Manually enable broadcast feature
+ try {
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "true");
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname,
+ "" + (5 * 1024));
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname,
+ "" + (2 * 1024));
+
+ runSimpleTests();
+
+ } finally {
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname,
+ TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.varname,
+ TajoConf.ConfVars.$DIST_QUERY_BROADCAST_NON_CROSS_JOIN_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.varname,
+ TajoConf.ConfVars.$DIST_QUERY_BROADCAST_CROSS_JOIN_THRESHOLD.defaultVal);
+ }
+ }
+
+ @SimpleTest
+ @Test
+ public void testTPCH_Q3() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testTPCH_Q5() throws Exception {
+ runSimpleTests();
+ }
+
+ // Predicates --------------------------------------------------------------
+
+ @SimpleTest
+ @Test
+ public void testSimpleFilter() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testInPredicateWithNumbers() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testInPredicateWithLiterals() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testBetweenNumbers() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testBetweenDates() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testCaseWhenFilter() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testFunctionWithinFilter() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testCountAsterisk() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testCtasToHdfs() throws Exception {
+ try {
+ executeString("CREATE DATABASE pgtmp").close();
+ executeString("CREATE TABLE pgtmp.offload AS SELECT * FROM LINEITEM").close();
+
+ runSimpleTests();
+
+ } finally {
+ executeString("DROP TABLE IF EXISTS pgtmp.offload").close();
+ executeString("DROP DATABASE IF EXISTS pgtmp").close();
+ }
+ }
+
+ @SimpleTest
+ @Test
+ public void testQueryWithConnProperties() throws Exception {
+ Map<String, String> connProperties = new HashMap<>();
+ connProperties.put("user", "postgres");
+ connProperties.put("password", "");
+
+ Optional<Tablespace> old = Optional.absent();
+ try {
+ old = PgSQLTestServer.resetAllParamsAndSetConnProperties(connProperties);
+ runSimpleTests();
+ } finally {
+ if (old.isPresent()) {
+ TablespaceManager.addTableSpaceForTest(old.get());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java
new file mode 100644
index 0000000..91aba63
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/TestPgSQLSimpleQueryTests.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import org.apache.tajo.QueryTestCaseBase;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestPgSQLSimpleQueryTests extends QueryTestCaseBase {
+ private static final String jdbcUrl = PgSQLTestServer.getInstance().getJdbcUrl();
+
+ public TestPgSQLSimpleQueryTests() {
+ super(PgSQLTestServer.DATABASE_NAME);
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ QueryTestCaseBase.testingCluster.getMaster().refresh();
+ }
+
+ @SimpleTest
+ @Test
+ public void testSelectAll() throws Exception {
+ runSimpleTests();
+ }
+
+ @SimpleTest
+ @Test
+ public void testSelectLimit() throws Exception {
+ runSimpleTests();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker b/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker
new file mode 100644
index 0000000..158780d
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker
@@ -0,0 +1 @@
+// for keeping dataset directory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/customer.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/customer.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/customer.sql
new file mode 100644
index 0000000..35b1861
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/customer.sql
@@ -0,0 +1,10 @@
+CREATE TABLE CUSTOMER (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME VARCHAR(25) NOT NULL,
+ C_ADDRESS VARCHAR(40) NOT NULL,
+ C_NATIONKEY INTEGER NOT NULL,
+ C_PHONE CHAR(15) NOT NULL,
+ C_ACCTBAL DECIMAL(15,2) NOT NULL,
+ C_MKTSEGMENT CHAR(10) NOT NULL,
+ C_COMMENT VARCHAR(117) NOT NULL
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.sql
new file mode 100644
index 0000000..9cbbc2f
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.sql
@@ -0,0 +1,5 @@
+CREATE TABLE DATETIME_TYPES (
+ date_field DATE,
+ time_field TIME,
+ ts_field TIMESTAMP
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.txt
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.txt b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.txt
new file mode 100644
index 0000000..6c7d8ff
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/datetime_types.txt
@@ -0,0 +1,4 @@
+1980-04-01|01:50:27|1980-04-01 01:50:27
+1984-01-05|02:45:31|1984-01-05 02:45:31
+1984-01-15|14:12:58|1984-01-15 14:12:58
+2015-03-21|08:11:01|2015-03-21 08:11:01
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/lineitem.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/lineitem.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/lineitem.sql
new file mode 100644
index 0000000..a7f61bb
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/lineitem.sql
@@ -0,0 +1,18 @@
+CREATE TABLE LINEITEM (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/nation.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/nation.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/nation.sql
new file mode 100644
index 0000000..f7ecda8
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/nation.sql
@@ -0,0 +1,6 @@
+CREATE TABLE NATION (
+ N_NATIONKEY INTEGER NOT NULL,
+ N_NAME CHAR(25) NOT NULL,
+ N_REGIONKEY INTEGER NOT NULL,
+ N_COMMENT VARCHAR(152)
+);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/orders.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/orders.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/orders.sql
new file mode 100644
index 0000000..220d576
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/orders.sql
@@ -0,0 +1,11 @@
+CREATE TABLE ORDERS (
+ O_ORDERKEY INTEGER NOT NULL,
+ O_CUSTKEY INTEGER NOT NULL,
+ O_ORDERSTATUS CHAR(1) NOT NULL,
+ O_TOTALPRICE DECIMAL(15,2) NOT NULL,
+ O_ORDERDATE DATE NOT NULL,
+ O_ORDERPRIORITY CHAR(15) NOT NULL,
+ O_CLERK CHAR(15) NOT NULL,
+ O_SHIPPRIORITY INTEGER NOT NULL,
+ O_COMMENT VARCHAR(79) NOT NULL
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/part.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/part.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/part.sql
new file mode 100644
index 0000000..e66f73c
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/part.sql
@@ -0,0 +1,11 @@
+CREATE TABLE PART (
+ P_PARTKEY INTEGER NOT NULL,
+ P_NAME VARCHAR(55) NOT NULL,
+ P_MFGR CHAR(25) NOT NULL,
+ P_BRAND CHAR(10) NOT NULL,
+ P_TYPE VARCHAR(25) NOT NULL,
+ P_SIZE INTEGER NOT NULL,
+ P_CONTAINER CHAR(10) NOT NULL,
+ P_RETAILPRICE DECIMAL(15,2) NOT NULL,
+ P_COMMENT VARCHAR(23) NOT NULL
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/partsupp.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/partsupp.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/partsupp.sql
new file mode 100644
index 0000000..1f61331
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/partsupp.sql
@@ -0,0 +1,7 @@
+CREATE TABLE PARTSUPP (
+ PS_PARTKEY INTEGER NOT NULL,
+ PS_SUPPKEY INTEGER NOT NULL,
+ PS_AVAILQTY INTEGER NOT NULL,
+ PS_SUPPLYCOST DECIMAL(15,2) NOT NULL,
+ PS_COMMENT VARCHAR(199) NOT NULL
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/region.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/region.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/region.sql
new file mode 100644
index 0000000..c47e26e
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/region.sql
@@ -0,0 +1,5 @@
+CREATE TABLE REGION (
+ R_REGIONKEY INTEGER NOT NULL,
+ R_NAME CHAR(25) NOT NULL,
+ R_COMMENT VARCHAR(152)
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/supplier.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/supplier.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/supplier.sql
new file mode 100644
index 0000000..a4d35d3
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/pgsql/supplier.sql
@@ -0,0 +1,9 @@
+CREATE TABLE SUPPLIER (
+ S_SUPPKEY INTEGER NOT NULL,
+ S_NAME CHAR(25) NOT NULL,
+ S_ADDRESS VARCHAR(40) NOT NULL,
+ S_NATIONKEY INTEGER NOT NULL,
+ S_PHONE CHAR(15) NOT NULL,
+ S_ACCTBAL DECIMAL(15,2) NOT NULL,
+ S_COMMENT VARCHAR(101) NOT NULL
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenDates.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenDates.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenDates.sql
new file mode 100644
index 0000000..e6b20a6
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenDates.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM WHERE L_SHIPDATE BETWEEN DATE '1996-01-1' AND DATE '1997-12-31';
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenNumbers.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenNumbers.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenNumbers.sql
new file mode 100644
index 0000000..e5d0408
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testBetweenNumbers.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM WHERE L_ORDERKEY BETWEEN 1 AND 2;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCaseWhenFilter.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCaseWhenFilter.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCaseWhenFilter.sql
new file mode 100644
index 0000000..2bdccde
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCaseWhenFilter.sql
@@ -0,0 +1,6 @@
+SELECT * FROM LINEITEM WHERE
+ CASE
+ WHEN L_RETURNFLAG = 'N' THEN TRUE
+ ELSE FALSE
+ END
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCountAsterisk.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCountAsterisk.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCountAsterisk.sql
new file mode 100644
index 0000000..fc9bc79
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCountAsterisk.sql
@@ -0,0 +1 @@
+SELECT COUNT(*) FROM LINEITEM;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCtasToHdfs.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCtasToHdfs.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCtasToHdfs.sql
new file mode 100644
index 0000000..843e46c
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testCtasToHdfs.sql
@@ -0,0 +1 @@
+SELECT * FROM pgtmp.offload;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testDateTimeTypes.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testDateTimeTypes.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testDateTimeTypes.sql
new file mode 100644
index 0000000..37c5377
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testDateTimeTypes.sql
@@ -0,0 +1 @@
+SELECT * FROM DATETIME_TYPES WHERE date_field > DATE '1980-01-01';
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFixedLengthFields.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFixedLengthFields.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFixedLengthFields.sql
new file mode 100644
index 0000000..fff0905
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFixedLengthFields.sql
@@ -0,0 +1 @@
+SELECT L_SHIPINSTRUCT, L_SHIPMODE FROM LINEITEM WHERE L_ORDERKEY > 1 AND L_LINENUMBER > 0;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFunctionWithinFilter.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFunctionWithinFilter.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFunctionWithinFilter.sql
new file mode 100644
index 0000000..61c6b3b
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testFunctionWithinFilter.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM WHERE CONCAT(L_RETURNFLAG, L_LINESTATUS) = 'NO'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithLiterals.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithLiterals.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithLiterals.sql
new file mode 100644
index 0000000..466f59e
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithLiterals.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM WHERE L_RETURNFLAG in ('N');
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithNumbers.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithNumbers.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithNumbers.sql
new file mode 100644
index 0000000..197858b
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testInPredicateWithNumbers.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM WHERE L_ORDERKEY in (1, 2);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testProjectedColumns.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testProjectedColumns.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testProjectedColumns.sql
new file mode 100644
index 0000000..1c7512b
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testProjectedColumns.sql
@@ -0,0 +1 @@
+SELECT L_ORDERKEY, L_SHIPDATE FROM LINEITEM WHERE L_ORDERKEY > 1 AND L_LINENUMBER > 0;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testQueryWithConnProperties.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testQueryWithConnProperties.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testQueryWithConnProperties.sql
new file mode 100644
index 0000000..a5eccc5
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testQueryWithConnProperties.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM WHERE L_ORDERKEY > 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testSimpleFilter.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testSimpleFilter.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testSimpleFilter.sql
new file mode 100644
index 0000000..a5eccc5
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testSimpleFilter.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM WHERE L_ORDERKEY > 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q1.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q1.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q1.sql
new file mode 100644
index 0000000..9e02d80
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q1.sql
@@ -0,0 +1,21 @@
+select
+ l_returnflag,
+ l_linestatus,
+ sum(l_quantity) as sum_qty,
+ sum(l_extendedprice) as sum_base_price,
+ sum(l_extendedprice*(1-l_discount)) as sum_disc_price,
+ sum(l_extendedprice*(1-l_discount)*(1+l_tax)) as sum_charge,
+ avg(l_quantity) as avg_qty,
+ avg(l_extendedprice) as avg_price,
+ avg(l_discount) as avg_disc,
+ count(*) as count_order
+from
+ lineitem
+where
+ l_shipdate <= DATE '1998-09-01'
+group by
+ l_returnflag,
+ l_linestatus
+order by
+ l_returnflag,
+ l_linestatus;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part.sql
new file mode 100644
index 0000000..ffaaf87
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part.sql
@@ -0,0 +1,18 @@
+select
+ s_acctbal,
+ s_name,
+ n_name,
+ p_partkey,
+ p_mfgr,
+ s_address,
+ s_phone,
+ s_comment,
+ ps_supplycost,
+ r_name,
+ p_type,
+ p_size
+from
+ region join nation on n_regionkey = r_regionkey and r_name = 'AMERICA'
+ join supplier on s_nationkey = n_nationkey
+ join partsupp on s_suppkey = ps_suppkey
+ join part on p_partkey = ps_partkey and p_type like '%BRASS' and p_size = 15;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.sql
new file mode 100644
index 0000000..e6ff43f
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.sql
@@ -0,0 +1,18 @@
+SELECT
+ s_acctbal,
+ s_name,
+ n_name,
+ p_partkey,
+ p_mfgr,
+ s_address,
+ s_phone,
+ s_comment,
+ ps_supplycost,
+ r_name,
+ p_type,
+ p_size
+FROM
+ default.region join tpch.nation on n_regionkey = r_regionkey and r_name = 'AMERICA'
+ join default.supplier on s_nationkey = n_nationkey
+ join default.partsupp on s_suppkey = ps_suppkey
+ join default.part on p_partkey = ps_partkey and p_type like '%BRASS' and p_size = 15;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q3.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q3.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q3.sql
new file mode 100644
index 0000000..7d338dd
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q3.sql
@@ -0,0 +1,22 @@
+select
+ l_orderkey,
+ sum(l_extendedprice*(1-l_discount)) as revenue,
+ o_orderdate,
+ o_shippriority
+from
+ customer,
+ orders,
+ lineitem
+where
+ c_mktsegment in ('BUILDING', 'AUTOMOBILE', 'MACHINERY', 'HOUSEHOLD') -- modified for selectivity
+ and c_custkey = o_custkey
+ and l_orderkey = o_orderkey
+ and o_orderdate < date '1996-12-31' -- modified for selectivity
+ and l_shipdate > date '1992-03-15' -- modified for selectivity
+group by
+ l_orderkey,
+ o_orderdate,
+ o_shippriority
+order by
+ revenue desc,
+ o_orderdate;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q5.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q5.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q5.sql
new file mode 100644
index 0000000..91e016b
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testTPCH_Q5.sql
@@ -0,0 +1,25 @@
+select n_name, revenue::float4 as revenue from (
+select
+ n_name,
+ sum(l_extendedprice * (1 - l_discount)) as revenue
+from
+ customer,
+ orders,
+ lineitem,
+ supplier,
+ nation,
+ region
+where
+ c_custkey = o_custkey and
+ /* l_orderkey = o_orderkey and -- modified for selectivity
+ l_suppkey = s_suppkey and
+ c_nationkey = s_nationkey and
+ s_nationkey = n_nationkey and
+ n_regionkey = r_regionkey and */
+ r_name = 'ASIA' and
+ o_orderdate >= date '1993-01-01' and o_orderdate < date '1997-01-01'
+group by
+ n_name
+order by
+ revenue desc
+) w;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testVariableLengthFields.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testVariableLengthFields.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testVariableLengthFields.sql
new file mode 100644
index 0000000..d72da1c
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLQueryTests/testVariableLengthFields.sql
@@ -0,0 +1 @@
+SELECT L_COMMENT FROM LINEITEM WHERE L_ORDERKEY > 1 AND L_LINENUMBER > 0;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectAll.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectAll.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectAll.sql
new file mode 100644
index 0000000..1ec1bf7
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectAll.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectLimit.sql
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectLimit.sql b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectLimit.sql
new file mode 100644
index 0000000..7173101
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/queries/TestPgSQLSimpleQueryTests/testSelectLimit.sql
@@ -0,0 +1 @@
+SELECT * FROM LINEITEM LIMIT 3;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenDates.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenDates.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenDates.result
new file mode 100644
index 0000000..a7f7d20
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenDates.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenNumbers.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenNumbers.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenNumbers.result
new file mode 100644
index 0000000..a7f7d20
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testBetweenNumbers.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCaseWhenFilter.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCaseWhenFilter.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCaseWhenFilter.result
new file mode 100644
index 0000000..a7f7d20
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCaseWhenFilter.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCountAsterisk.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCountAsterisk.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCountAsterisk.result
new file mode 100644
index 0000000..7044801
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCountAsterisk.result
@@ -0,0 +1,3 @@
+?count
+-------------------------------
+5
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCtasToHdfs.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCtasToHdfs.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCtasToHdfs.result
new file mode 100644
index 0000000..dd34c54
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testCtasToHdfs.result
@@ -0,0 +1,7 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
+3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco
+3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL,unusual accounts. eve
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testDateTimeTypes.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testDateTimeTypes.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testDateTimeTypes.result
new file mode 100644
index 0000000..cd68a38
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testDateTimeTypes.result
@@ -0,0 +1,6 @@
+date_field,time_field,ts_field
+-------------------------------
+1980-04-01,01:50:27,1980-04-01 01:50:27
+1984-01-05,02:45:31,1984-01-05 02:45:31
+1984-01-15,14:12:58,1984-01-15 14:12:58
+2015-03-21,08:11:01,2015-03-21 08:11:01
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFixedLengthFields.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFixedLengthFields.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFixedLengthFields.result
new file mode 100644
index 0000000..ae8384d
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFixedLengthFields.result
@@ -0,0 +1,5 @@
+l_shipinstruct,l_shipmode
+-------------------------------
+TAKE BACK RETURN,RAIL
+NONE,AIR
+TAKE BACK RETURN,RAIL
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFunctionWithinFilter.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFunctionWithinFilter.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFunctionWithinFilter.result
new file mode 100644
index 0000000..a7f7d20
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testFunctionWithinFilter.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithLiterals.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithLiterals.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithLiterals.result
new file mode 100644
index 0000000..a7f7d20
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithLiterals.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithNumbers.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithNumbers.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithNumbers.result
new file mode 100644
index 0000000..a7f7d20
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testInPredicateWithNumbers.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testProjectedColumns.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testProjectedColumns.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testProjectedColumns.result
new file mode 100644
index 0000000..bb3f58d
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testProjectedColumns.result
@@ -0,0 +1,5 @@
+l_orderkey,l_shipdate
+-------------------------------
+2,1997-01-28
+3,1994-02-02
+3,1993-11-09
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testQueryWithConnProperties.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testQueryWithConnProperties.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testQueryWithConnProperties.result
new file mode 100644
index 0000000..2eb253e
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testQueryWithConnProperties.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
+3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco
+3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL,unusual accounts. eve
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testSimpleFilter.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testSimpleFilter.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testSimpleFilter.result
new file mode 100644
index 0000000..2eb253e
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testSimpleFilter.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
+3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco
+3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL,unusual accounts. eve
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q1.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q1.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q1.result
new file mode 100644
index 0000000..3020f56
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q1.result
@@ -0,0 +1,4 @@
+l_returnflag,l_linestatus,sum_qty,sum_base_price,sum_disc_price,sum_charge,avg_qty,avg_price,avg_disc,count_order
+-------------------------------
+N,O,91.0,111845.85,106860.63639999999,112012.46995200001,30.333333333333332,37281.950000000004,0.043333333333333335,3
+R,F,94.0,100854.52,92931.39000000001,92931.39000000001,47.0,50427.26,0.08,2
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part.result
new file mode 100644
index 0000000..64d8e12
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part.result
@@ -0,0 +1,3 @@
+s_acctbal,s_name,n_name,p_partkey,p_mfgr,s_address,s_phone,s_comment,ps_supplycost,r_name,p_type,p_size
+-------------------------------
+4192.4,Supplier#000000003,ARGENTINA,2,Manufacturer#1,q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3,11-383-516-1199,blithely silent requests after the express dependencies are sl,1.01,AMERICA,LARGE BRUSHED BRASS,15
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.result
new file mode 100644
index 0000000..64d8e12
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q2_Part_MixedStorage.result
@@ -0,0 +1,3 @@
+s_acctbal,s_name,n_name,p_partkey,p_mfgr,s_address,s_phone,s_comment,ps_supplycost,r_name,p_type,p_size
+-------------------------------
+4192.4,Supplier#000000003,ARGENTINA,2,Manufacturer#1,q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3,11-383-516-1199,blithely silent requests after the express dependencies are sl,1.01,AMERICA,LARGE BRUSHED BRASS,15
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q3.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q3.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q3.result
new file mode 100644
index 0000000..2ad8a05
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q3.result
@@ -0,0 +1,5 @@
+l_orderkey,revenue,o_orderdate,o_shippriority
+-------------------------------
+3,92931.39000000001,1993-10-14,0
+1,62166.1764,1996-01-02,0
+2,44694.46,1996-12-01,0
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q5.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q5.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q5.result
new file mode 100644
index 0000000..09f3cad
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testTPCH_Q5.result
@@ -0,0 +1,27 @@
+n_name,revenue
+-------------------------------
+ROMANIA,1798128.2
+SAUDI ARABIA,1798128.2
+IRAQ,1798128.2
+UNITED STATES,1798128.2
+EGYPT,1798128.2
+ETHIOPIA,1798128.2
+ALGERIA,1798128.2
+IRAN,1798128.2
+MOZAMBIQUE,1798128.2
+KENYA,1798128.2
+VIETNAM,1798128.2
+MOROCCO,1798128.2
+GERMANY,1798128.2
+PERU,1798128.2
+CANADA,1798128.2
+INDONESIA,1798128.2
+JAPAN,1798128.2
+INDIA,1798128.2
+CHINA,1798128.2
+FRANCE,1798128.2
+UNITED KINGDOM,1798128.2
+BRAZIL,1798128.2
+RUSSIA,1798128.2
+JORDAN,1798128.2
+ARGENTINA,1798128.2
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testVariableLengthFields.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testVariableLengthFields.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testVariableLengthFields.result
new file mode 100644
index 0000000..531402c
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLQueryTests/testVariableLengthFields.result
@@ -0,0 +1,5 @@
+l_comment
+-------------------------------
+ven requests. deposits breach a
+ongside of the furiously brave acco
+unusual accounts. eve
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result
new file mode 100644
index 0000000..dd34c54
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectAll.result
@@ -0,0 +1,7 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
+3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco
+3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL,unusual accounts. eve
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result
new file mode 100644
index 0000000..a7f7d20
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/results/TestPgSQLSimpleQueryTests/testSelectLimit.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
[2/4] tajo git commit: TAJO-1730: JDBC Tablespace support.
Posted by hy...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml
new file mode 100644
index 0000000..35000a4
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/pom.xml
@@ -0,0 +1,243 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tajo-project</artifactId>
+ <groupId>org.apache.tajo</groupId>
+ <version>0.12.0-SNAPSHOT</version>
+ <relativePath>../../tajo-project</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tajo-storage-jdbc</artifactId>
+ <packaging>jar</packaging>
+ <name>Tajo JDBC storage common</name>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>repository.jboss.org</id>
+ <url>https://repository.jboss.org/nexus/content/repositories/releases/
+ </url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ <encoding>${project.build.sourceEncoding}</encoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/*.sql</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemProperties>
+ <tajo.test>TRUE</tajo.test>
+ </systemProperties>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-protobuf-generated-sources-directory</id>
+ <phase>initialize</phase>
+ <configuration>
+ <target>
+ <mkdir dir="target/generated-sources/proto" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2</version>
+ <executions>
+ <execution>
+ <id>generate-sources</id>
+ <phase>generate-sources</phase>
+ <configuration>
+ <executable>protoc</executable>
+ <arguments>
+ <argument>-Isrc/main/proto/</argument>
+ <argument>--proto_path=../../tajo-common/src/main/proto</argument>
+ <argument>--proto_path=../../tajo-catalog/tajo-catalog-common/src/main/proto</argument>
+ <argument>--java_out=target/generated-sources/proto</argument>
+ <argument>src/main/proto/JdbcFragmentProtos.proto</argument>
+ </arguments>
+ </configuration>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.5</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/proto</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-catalog-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-plan</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>docs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <!-- build javadoc jars per jar for publishing to maven -->
+ <id>module-javadocs</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <destDir>${project.build.directory}</destDir>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <version>2.15</version>
+ </plugin>
+ </plugins>
+ </reporting>
+</project>
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java
new file mode 100644
index 0000000..2910ea5
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/ConnectionInfo.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import org.apache.tajo.exception.TajoInternalError;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ConnectionInfo {
+ String scheme;
+ String host;
+ String dbName;
+ String tableName;
+ String user;
+ String password;
+ Map<String, String> params;
+
+ public String scheme() {
+ return scheme;
+ }
+
+ public String database() {
+ return dbName;
+ }
+
+ public String table() {
+ return tableName;
+ }
+
+ public String user() {
+ return user;
+ }
+
+ public String password() {
+ return password;
+ }
+
+ public static ConnectionInfo fromURI(String originalUri) {
+ return fromURI(URI.create(originalUri));
+ }
+
+ public static ConnectionInfo fromURI(URI originalUri) {
+ final String uriStr = originalUri.toASCIIString();
+ URI uri = originalUri;
+
+ final ConnectionInfo connInfo = new ConnectionInfo();
+ connInfo.scheme = uriStr.substring(0, uriStr.indexOf("://"));
+
+ if (connInfo.scheme.split(":").length > 1) {
+ int idx = uriStr.indexOf(':');
+ uri = URI.create(uriStr.substring(idx + 1));
+ }
+
+ connInfo.host = uri.getHost();
+
+ String path = uri.getPath();
+ if (path != null && !path.isEmpty()) {
+ String [] pathElements = path.substring(1).split("/");
+ if (pathElements.length != 1) {
+ throw new TajoInternalError("Invalid JDBC path: " + path);
+ }
+ connInfo.dbName = pathElements[0];
+ }
+
+ Map<String, String> params = new HashMap<>();
+
+ int paramIndex = uriStr.indexOf("?");
+ if (paramIndex > 0) {
+ String parameterPart = uriStr.substring(paramIndex+1, uriStr.length());
+
+ String [] eachParam = parameterPart.split("&");
+
+ for (String each: eachParam) {
+ String [] keyValues = each.split("=");
+ if (keyValues.length != 2) {
+ throw new TajoInternalError("Invalid URI Parameters: " + parameterPart);
+ }
+ params.put(keyValues[0], keyValues[1]);
+ }
+ }
+
+ if (params.containsKey("table")) {
+ connInfo.tableName = params.remove("table");
+ }
+
+ if (params.containsKey("user")) {
+ connInfo.user = params.remove("user");
+ }
+ if (params.containsKey("password")) {
+ connInfo.password = params.remove("password");
+ }
+
+ connInfo.params = params;
+
+ return connInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java
new file mode 100644
index 0000000..bf9536e
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.storage.fragment.Fragment;
+import org.apache.tajo.storage.jdbc.JdbcFragmentProtos.JdbcFragmentProto;
+import org.apache.tajo.util.TUtil;
+
+public class JdbcFragment implements Fragment, Comparable<JdbcFragment>, Cloneable {
+ String uri;
+ String inputSourceId;
+ String [] hostNames;
+
+
+ public JdbcFragment(ByteString raw) throws InvalidProtocolBufferException {
+ JdbcFragmentProto.Builder builder = JdbcFragmentProto.newBuilder();
+ builder.mergeFrom(raw);
+ builder.build();
+ init(builder.build());
+ }
+
+ public JdbcFragment(String inputSourceId, String uri) {
+ this.inputSourceId = inputSourceId;
+ this.uri = uri;
+ this.hostNames = extractHosts(uri);
+ }
+
+ private void init(JdbcFragmentProto proto) {
+ this.uri = proto.getUri();
+ this.inputSourceId = proto.getInputSourceId();
+ this.hostNames = proto.getHostsList().toArray(new String [proto.getHostsCount()]);
+ }
+
+ private String [] extractHosts(String uri) {
+ return new String[] {ConnectionInfo.fromURI(uri).host};
+ }
+
+ @Override
+ public String getTableName() {
+ return inputSourceId;
+ }
+
+ public String getUri() {
+ return uri;
+ }
+
+ @Override
+ public long getLength() {
+ return 0;
+ }
+
+ @Override
+ public String getKey() {
+ return null;
+ }
+
+ @Override
+ public String[] getHosts() {
+ return hostNames;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return false;
+ }
+
+ @Override
+ public CatalogProtos.FragmentProto getProto() {
+ JdbcFragmentProto.Builder builder = JdbcFragmentProto.newBuilder();
+ builder.setInputSourceId(this.inputSourceId);
+ builder.setUri(this.uri);
+ if(hostNames != null) {
+ builder.addAllHosts(TUtil.newList(hostNames));
+ }
+
+ CatalogProtos.FragmentProto.Builder fragmentBuilder = CatalogProtos.FragmentProto.newBuilder();
+ fragmentBuilder.setId(this.inputSourceId);
+ fragmentBuilder.setStoreType("JDBC");
+ fragmentBuilder.setContents(builder.buildPartial().toByteString());
+ return fragmentBuilder.build();
+ }
+
+ @Override
+ public int compareTo(JdbcFragment o) {
+ return this.uri.compareTo(o.uri);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java
new file mode 100644
index 0000000..eff1b9c
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.exception.*;
+import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.Pair;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+import java.sql.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType;
+
+public abstract class JdbcMetadataProviderBase implements MetadataProvider {
+ protected static final Log LOG = LogFactory.getLog(JdbcMetadataProviderBase.class);
+
+ public static String [] GENERAL_TABLE_TYPES = new String [] {"TABLE"};
+
+ protected final JdbcTablespace space;
+ protected final String databaseName;
+
+ protected final String jdbcUri;
+
+ protected final Connection connection;
+
+ public JdbcMetadataProviderBase(JdbcTablespace space, String dbName) {
+ this.space = space;
+ this.databaseName = dbName;
+
+ ConnectionInfo connInfo = ConnectionInfo.fromURI(space.getUri());
+ this.jdbcUri = space.getUri().toASCIIString();
+
+ try {
+ Class.forName(getJdbcDriverName()).newInstance();
+ LOG.info(getJdbcDriverName() + " is loaded...");
+ } catch (Exception e) {
+ throw new TajoInternalError(e);
+ }
+
+ try {
+ connection = DriverManager.getConnection(jdbcUri, space.connProperties);
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ }
+ }
+
+ @Override
+ public String getTablespaceName() {
+ return space.getName();
+ }
+
+ @Override
+ public URI getTablespaceUri() {
+ return space.getUri();
+ }
+
+ @Override
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ @Override
+ public Collection<String> getSchemas() {
+ return Collections.EMPTY_SET;
+ }
+
+ @Override
+ public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) {
+ ResultSet res = null;
+ List<String> tableNames = Lists.newArrayList();
+ try {
+ res = connection.getMetaData().getTables(databaseName, schemaPattern, tablePattern, GENERAL_TABLE_TYPES);
+ while(res.next()) {
+ tableNames.add(res.getString("TABLE_NAME"));
+ }
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ } finally {
+ try {
+ if (res != null) {
+ res.close();
+ }
+ } catch (SQLException e) {
+ LOG.warn(e);
+ }
+ }
+
+ return tableNames;
+ }
+
+ private TypeDesc convertDataType(ResultSet res) throws SQLException {
+ final int typeId = res.getInt("DATA_TYPE");
+
+ switch (typeId ) {
+ case Types.BOOLEAN:
+ return new TypeDesc(newSimpleDataType(Type.BOOLEAN));
+
+ case Types.TINYINT:
+ case Types.SMALLINT:
+ case Types.INTEGER:
+ return new TypeDesc(newSimpleDataType(Type.INT4));
+
+ case Types.DISTINCT: // sequence for postgresql
+ case Types.BIGINT:
+ return new TypeDesc(newSimpleDataType(Type.INT8));
+
+ case Types.FLOAT:
+ return new TypeDesc(newSimpleDataType(Type.FLOAT4));
+
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ case Types.DOUBLE:
+ return new TypeDesc(newSimpleDataType(Type.FLOAT8));
+
+ case Types.DATE:
+ return new TypeDesc(newSimpleDataType(Type.DATE));
+
+ case Types.TIME:
+ return new TypeDesc(newSimpleDataType(Type.TIME));
+
+ case Types.TIMESTAMP:
+ return new TypeDesc(newSimpleDataType(Type.TIMESTAMP));
+
+ case Types.CHAR:
+ case Types.NCHAR:
+ case Types.VARCHAR:
+ case Types.NVARCHAR:
+ case Types.CLOB:
+ case Types.NCLOB:
+ case Types.LONGVARCHAR:
+ case Types.LONGNVARCHAR:
+ return new TypeDesc(newSimpleDataType(Type.TEXT));
+
+ case Types.BINARY:
+ case Types.VARBINARY:
+ case Types.BLOB:
+ return new TypeDesc(newSimpleDataType(Type.BLOB));
+
+ default:
+ throw SQLExceptionUtil.toSQLException(new UnsupportedDataTypeException(typeId + ""));
+ }
+ }
+
+ @Override
+ public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException {
+ ResultSet resultForTable = null;
+ ResultSet resultForColumns = null;
+ try {
+
+ // get table name
+ resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, null);
+
+ if (!resultForTable.next()) {
+ throw new UndefinedTablespaceException(tableName);
+ }
+ final String name = resultForTable.getString("TABLE_NAME");
+
+ // get columns
+ resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null);
+
+ final List<Pair<Integer, Column>> columns = Lists.newArrayList();
+
+ while(resultForColumns.next()) {
+ final int ordinalPos = resultForColumns.getInt("ORDINAL_POSITION");
+ final String qualifier = resultForColumns.getString("TABLE_NAME");
+ final String columnName = resultForColumns.getString("COLUMN_NAME");
+ final TypeDesc type = convertDataType(resultForColumns);
+ final Column c = new Column(CatalogUtil.buildFQName(databaseName, qualifier, columnName), type);
+
+ columns.add(new Pair<>(ordinalPos, c));
+ }
+
+ // sort columns in an order of ordinal position
+ Collections.sort(columns, new Comparator<Pair<Integer, Column>>() {
+ @Override
+ public int compare(Pair<Integer, Column> o1, Pair<Integer, Column> o2) {
+ return o1.getFirst() - o2.getFirst();
+ }
+ });
+
+ // transform the pair list into collection for columns
+ final Schema schema = new Schema(Collections2.transform(columns, new Function<Pair<Integer,Column>, Column>() {
+ @Override
+ public Column apply(@Nullable Pair<Integer, Column> columnPair) {
+ return columnPair.getSecond();
+ }
+ }));
+
+
+ // fill the table stats
+ final TableStats stats = new TableStats();
+ stats.setNumRows(-1); // unknown
+
+ final TableDesc table = new TableDesc(
+ CatalogUtil.buildFQName(databaseName, name),
+ schema,
+ new TableMeta("rowstore", new KeyValueSet()),
+ space.getTableUri(databaseName, name)
+ );
+ table.setStats(stats);
+
+ return table;
+
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ } finally {
+ try {
+ if (resultForTable != null) {
+ resultForTable.close();
+ }
+
+ if (resultForColumns != null) {
+ resultForColumns.close();
+ }
+
+ } catch (SQLException e) {
+ LOG.warn(e);
+ }
+ }
+ }
+
+ protected abstract String getJdbcDriverName();
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcScanner.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcScanner.java
new file mode 100644
index 0000000..82c3be3
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcScanner.java
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.TimeDatum;
+import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.exception.TajoRuntimeException;
+import org.apache.tajo.exception.UnsupportedDataTypeException;
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.LogicalNode;
+import org.apache.tajo.storage.Scanner;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.VTuple;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.sql.*;
+import java.util.Iterator;
+import java.util.Properties;
+
+public abstract class JdbcScanner implements Scanner {
+ private static final Log LOG = LogFactory.getLog(JdbcScanner.class);
+
+ protected final DatabaseMetaData dbMetaData;
+ /** JDBC Connection Properties */
+ protected final Properties connProperties;
+ protected final String tableName;
+ protected final Schema schema;
+ protected final TableMeta tableMeta;
+ protected final JdbcFragment fragment;
+ protected final TableStats stats;
+ protected final SQLBuilder builder;
+
+ protected Column [] targets;
+ protected EvalNode filter;
+ protected Long limit;
+ protected LogicalNode planPart;
+ protected VTuple outTuple;
+ protected String generatedSql;
+ protected ResultSetIterator iter;
+
+ protected int recordCount = 0;
+
+ /**
+ *
+ * @param dbMetaData DatabaseMetaData
+ * @param connProperties JDBC Connection Properties
+ * @param tableSchema Table Schema
+ * @param tableMeta Table Properties
+ * @param fragment Fragment
+ */
+ public JdbcScanner(final DatabaseMetaData dbMetaData,
+ final Properties connProperties,
+ final Schema tableSchema,
+ final TableMeta tableMeta,
+ final JdbcFragment fragment) {
+
+ Preconditions.checkNotNull(dbMetaData);
+ Preconditions.checkNotNull(connProperties);
+ Preconditions.checkNotNull(tableSchema);
+ Preconditions.checkNotNull(tableMeta);
+ Preconditions.checkNotNull(fragment);
+
+ this.dbMetaData = dbMetaData;
+ this.connProperties = connProperties;
+ this.tableName = ConnectionInfo.fromURI(fragment.getUri()).tableName;
+ this.schema = tableSchema;
+ this.tableMeta = tableMeta;
+ this.fragment = fragment;
+ this.stats = new TableStats();
+ builder = getSQLBuilder();
+ }
+
+ @Override
+ public void init() throws IOException {
+ if (targets == null) {
+ targets = schema.toArray();
+ }
+ outTuple = new VTuple(targets.length);
+
+ if (planPart == null) {
+ generatedSql = builder.build(tableName, targets, filter, limit);
+ } else {
+ generatedSql = builder.build(planPart);
+ }
+ }
+
+ @Override
+ public Tuple next() throws IOException {
+ if (iter == null) {
+ iter = executeQueryAndGetIter();
+ }
+
+ if (iter.hasNext()) {
+ return iter.next();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public void reset() throws IOException {
+ if (iter != null) {
+ iter.rewind();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (iter != null) {
+ iter.close();
+ }
+ }
+
+ @Override
+ public void pushOperators(LogicalNode planPart) {
+ this.planPart = planPart;
+ }
+
+
+ @Override
+ public boolean isProjectable() {
+ return true;
+ }
+
+ @Override
+ public void setTarget(Column [] targets) {
+ this.targets = targets;
+ }
+
+ @Override
+ public boolean isSelectable() {
+ return true;
+ }
+
+ @Override
+ public void setFilter(EvalNode filter) {
+ this.filter = filter;
+ }
+
+ @Override
+ public void setLimit(long num) {
+ this.limit = num;
+ }
+
+ @Override
+ public boolean isSplittable() {
+ return false;
+ }
+
+ @Override
+ public float getProgress() {
+ return 0;
+ }
+
+ @Override
+ public TableStats getInputStats() {
+ return stats;
+ }
+
+ @Override
+ public Schema getSchema() {
+ return schema;
+ }
+
+ protected SQLBuilder getSQLBuilder() {
+ return new SQLBuilder(dbMetaData, getSQLExprBuilder());
+ }
+ protected SQLExpressionGenerator getSQLExprBuilder() {
+ return new SQLExpressionGenerator(dbMetaData);
+ }
+
+ protected void convertTuple(ResultSet resultSet, VTuple tuple) {
+ try {
+ for (int column_idx = 0; column_idx < targets.length; column_idx++) {
+ final Column c = targets[column_idx];
+ final int resultIdx = column_idx + 1;
+
+ switch (c.getDataType().getType()) {
+ case INT1:
+ case INT2:
+ tuple.put(column_idx, DatumFactory.createInt2(resultSet.getShort(resultIdx)));
+ break;
+ case INT4:
+ tuple.put(column_idx, DatumFactory.createInt4(resultSet.getInt(resultIdx)));
+ break;
+ case INT8:
+ tuple.put(column_idx, DatumFactory.createInt8(resultSet.getLong(resultIdx)));
+ break;
+ case FLOAT4:
+ tuple.put(column_idx, DatumFactory.createFloat4(resultSet.getFloat(resultIdx)));
+ break;
+ case FLOAT8:
+ tuple.put(column_idx, DatumFactory.createFloat8(resultSet.getDouble(resultIdx)));
+ break;
+ case CHAR:
+ tuple.put(column_idx, DatumFactory.createText(resultSet.getString(resultIdx)));
+ break;
+ case VARCHAR:
+ case TEXT:
+ // TODO - trim is unnecessary in many cases, so we can use it for certain cases
+ tuple.put(column_idx, DatumFactory.createText(resultSet.getString(resultIdx).trim()));
+ break;
+ case DATE:
+ final Date date = resultSet.getDate(resultIdx);
+ tuple.put(column_idx, DatumFactory.createDate(1900 + date.getYear(), 1 + date.getMonth(), date.getDate()));
+ break;
+ case TIME:
+ tuple.put(column_idx, new TimeDatum(resultSet.getTime(resultIdx).getTime() * 1000));
+ break;
+ case TIMESTAMP:
+ tuple.put(column_idx,
+ DatumFactory.createTimestmpDatumWithJavaMillis(resultSet.getTimestamp(resultIdx).getTime()));
+ break;
+ case BINARY:
+ case VARBINARY:
+ case BLOB:
+ tuple.put(column_idx,
+ DatumFactory.createBlob(resultSet.getBytes(resultIdx)));
+ break;
+ default:
+ throw new TajoInternalError(new UnsupportedDataTypeException(c.getDataType().getType().name()));
+ }
+ }
+ } catch (SQLException s) {
+ throw new TajoInternalError(s);
+ }
+ }
+
+ private ResultSetIterator executeQueryAndGetIter() {
+ try {
+ LOG.info("Generated SQL: " + generatedSql);
+ Connection conn = DriverManager.getConnection(fragment.uri, connProperties);
+ Statement statement = conn.createStatement();
+ ResultSet resultset = statement.executeQuery(generatedSql);
+ return new ResultSetIterator((resultset));
+ } catch (SQLException s) {
+ throw new TajoInternalError(s);
+ }
+ }
+
+ public class ResultSetIterator implements Iterator<Tuple>, Closeable {
+
+ private final ResultSet resultSet;
+
+ private boolean didNext = false;
+ private boolean hasNext = false;
+
+ public ResultSetIterator(ResultSet resultSet) {
+ this.resultSet = resultSet;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!didNext) {
+
+ try {
+ hasNext = resultSet.next();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+
+ didNext = true;
+ }
+ return hasNext;
+ }
+
+ @Override
+ public Tuple next() {
+ if (!didNext) {
+ try {
+ resultSet.next();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ didNext = false;
+ convertTuple(resultSet, outTuple);
+ recordCount++;
+ return outTuple;
+ }
+
+ @Override
+ public void remove() {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ public void rewind() {
+ try {
+ resultSet.isBeforeFirst();
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ resultSet.close();
+ } catch (SQLException e) {
+ LOG.warn(e);
+ }
+
+ if (stats != null) {
+ stats.setNumRows(recordCount);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
new file mode 100644
index 0000000..7489307
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import net.minidev.json.JSONObject;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.exception.NotImplementedException;
+import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.exception.TajoRuntimeException;
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.LogicalNode;
+import org.apache.tajo.storage.*;
+import org.apache.tajo.storage.fragment.Fragment;
+import org.apache.tajo.util.UriUtil;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * JDBC Tablespace
+ */
+public abstract class JdbcTablespace extends Tablespace {
+ private static final Log LOG = LogFactory.getLog(JdbcTablespace.class);
+
+ static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true);
+ static final FormatProperty FORMAT_PROPERTY = new FormatProperty(false, false, false);
+
+ /**
+ * required configuration
+ */
+ public static final String CONFIG_KEY_MAPPED_DATABASE = "mapped_database";
+ /**
+ * optional configuration
+ */
+ public static final String CONFIG_KEY_CONN_PROPERTIES = "connection_properties";
+
+ public static final String URI_PARAM_KEY_TABLE = "table";
+
+ protected Connection conn;
+ protected String database;
+ protected Properties connProperties = new Properties();
+
+ public JdbcTablespace(String name, URI uri, JSONObject config) {
+ super(name, uri, config);
+ setDatabase();
+ setJdbcProperties();
+ }
+
+ private void setDatabase() {
+ if (config.containsKey(CONFIG_KEY_MAPPED_DATABASE)) {
+ database = this.config.getAsString(CONFIG_KEY_MAPPED_DATABASE);
+ } else {
+ database = ConnectionInfo.fromURI(uri).database();
+ }
+ }
+
+ private void setJdbcProperties() {
+ Object connPropertiesObjects = config.get(CONFIG_KEY_CONN_PROPERTIES);
+ if (connPropertiesObjects != null) {
+ Preconditions.checkState(connPropertiesObjects instanceof JSONObject, "Invalid jdbc_properties field in configs");
+ JSONObject connProperties = (JSONObject) connPropertiesObjects;
+
+ for (Map.Entry<String, Object> entry : connProperties.entrySet()) {
+ this.connProperties.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ @Override
+ protected void storageInit() throws IOException {
+ try {
+ this.conn = DriverManager.getConnection(uri.toASCIIString(), connProperties);
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public long getTableVolume(URI uri) throws UnsupportedException {
+ throw new UnsupportedException();
+ }
+
+ @Override
+ public URI getTableUri(String databaseName, String tableName) {
+ return URI.create(UriUtil.addParam(getUri().toASCIIString(), URI_PARAM_KEY_TABLE, tableName));
+ }
+
+ @Override
+ public List<Fragment> getSplits(String inputSourceId,
+ TableDesc tableDesc,
+ @Nullable EvalNode filterCondition) throws IOException {
+ return Lists.newArrayList((Fragment)new JdbcFragment(inputSourceId, tableDesc.getUri().toASCIIString()));
+ }
+
+ @Override
+ public StorageProperty getProperty() {
+ return STORAGE_PROPERTY;
+ }
+
+ @Override
+ public FormatProperty getFormatProperty(TableMeta meta) {
+ return FORMAT_PROPERTY;
+ }
+
+ @Override
+ public void close() {
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ LOG.warn(e);
+ }
+ }
+ }
+
+ @Override
+ public TupleRange[] getInsertSortRanges(OverridableConf queryContext,
+ TableDesc tableDesc,
+ Schema inputSchema,
+ SortSpec[] sortSpecs,
+ TupleRange dataRange) throws IOException {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ @Override
+ public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ @Override
+ public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ @Override
+ public void purgeTable(TableDesc tableDesc) throws IOException {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ @Override
+ public void prepareTable(LogicalNode node) throws IOException {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ @Override
+ public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema,
+ TableDesc tableDesc) throws IOException {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ @Override
+ public void rollbackTable(LogicalNode node) throws IOException {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ @Override
+ public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ public abstract MetadataProvider getMetadataProvider();
+
+ @Override
+ public abstract Scanner getScanner(TableMeta meta,
+ Schema schema,
+ Fragment fragment,
+ @Nullable Schema target) throws IOException;
+
+ public DatabaseMetaData getDatabaseMetaData() {
+ try {
+ return conn.getMetaData();
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLBuilder.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLBuilder.java
new file mode 100644
index 0000000..f72d6ee
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLBuilder.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import org.apache.tajo.catalog.Column;
+import org.apache.tajo.exception.TajoRuntimeException;
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.Target;
+import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.util.StringUtils;
+
+import javax.annotation.Nullable;
+import java.sql.DatabaseMetaData;
+import java.util.Stack;
+
+/**
+ * Generator to build a SQL statement from a plan fragment
+ */
+public class SQLBuilder {
+ @SuppressWarnings("unused")
+ private final DatabaseMetaData dbMetaData;
+ private final SQLExpressionGenerator sqlExprGen;
+
+ public static class SQLBuilderContext {
+ StringBuilder sb;
+ }
+
+ public SQLBuilder(DatabaseMetaData dbMetaData, SQLExpressionGenerator exprGen) {
+ this.dbMetaData = dbMetaData;
+ this.sqlExprGen = exprGen;
+ }
+
+ public String build(String tableName, Column [] targets, @Nullable EvalNode filter, @Nullable Long limit) {
+
+ StringBuilder selectClause = new StringBuilder("SELECT ");
+ if (targets.length > 0) {
+ selectClause.append(StringUtils.join(targets, ",", new Function<Column, String>() {
+ @Override
+ public String apply(@Nullable Column input) {
+ return input.getSimpleName();
+ }
+ }));
+ } else {
+ selectClause.append("1");
+ }
+ selectClause.append(" ");
+
+ StringBuilder fromClause = new StringBuilder("FROM ");
+ fromClause.append(tableName).append(" ");
+
+ StringBuilder whereClause = null;
+ if (filter != null) {
+ whereClause = new StringBuilder("WHERE ");
+ whereClause.append(sqlExprGen.generate(filter)).append(" ");
+ }
+
+ StringBuilder limitClause = null;
+ if (limit != null) {
+ limitClause = new StringBuilder("LIMIT ");
+ limitClause.append(limit).append(" ");
+ }
+
+ return generateSelectStmt(selectClause, fromClause, whereClause, limitClause);
+ }
+
+ public String generateSelectStmt(StringBuilder selectClause,
+ StringBuilder fromClause,
+ @Nullable StringBuilder whereClause,
+ @Nullable StringBuilder limitClause) {
+ return
+ selectClause.toString() +
+ fromClause.toString() +
+ (whereClause != null ? whereClause.toString() : "") +
+ (limitClause != null ? limitClause.toString() : "");
+ }
+
+ public String build(LogicalNode planPart) {
+ SQLBuilderContext context = new SQLBuilderContext();
+ visit(context, planPart, new Stack<LogicalNode>());
+ return context.sb.toString();
+ }
+
+ public void visit(SQLBuilderContext context, LogicalNode node, Stack<LogicalNode> stack) {
+ stack.push(node);
+
+ switch (node.getType()) {
+ case SCAN:
+ visitScan(context, (ScanNode) node, stack);
+ break;
+
+ case GROUP_BY:
+ visitGroupBy(context, (GroupbyNode) node, stack);
+ break;
+
+ case SELECTION:
+ visitFilter(context, (SelectionNode) node, stack);
+ break;
+
+ case PROJECTION:
+ visitProjection(context, (ProjectionNode) node, stack);
+ break;
+
+ case TABLE_SUBQUERY:
+ visitDerivedSubquery(context, (TableSubQueryNode) node, stack);
+ break;
+
+ default:
+ throw new TajoRuntimeException(new UnsupportedException("plan node '" + node.getType().name() + "'"));
+ }
+
+ stack.pop();
+ }
+
+ public void visitDerivedSubquery(SQLBuilderContext ctx, TableSubQueryNode derivedSubquery, Stack<LogicalNode> stack) {
+ ctx.sb.append(" (");
+ visit(ctx, derivedSubquery.getSubQuery(), stack);
+ ctx.sb.append(" ) ").append(derivedSubquery.getTableName());
+ }
+
+ public void visitProjection(SQLBuilderContext ctx, ProjectionNode projection, Stack<LogicalNode> stack) {
+
+ visit(ctx, projection.getChild(), stack);
+ }
+
+ public void visitGroupBy(SQLBuilderContext ctx, GroupbyNode groupby, Stack<LogicalNode> stack) {
+ visit(ctx, groupby.getChild(), stack);
+ ctx.sb.append("GROUP BY ").append(StringUtils.join(groupby.getGroupingColumns(), ",", 0)).append(" ");
+ }
+
+ public void visitFilter(SQLBuilderContext ctx, SelectionNode filter, Stack<LogicalNode> stack) {
+ visit(ctx, filter.getChild(), stack);
+ ctx.sb.append("WHERE " + sqlExprGen.generate(filter.getQual()));
+ }
+
+ public void visitScan(SQLBuilderContext ctx, ScanNode scan, Stack<LogicalNode> stack) {
+
+ StringBuilder selectClause = new StringBuilder("SELECT ");
+ if (scan.getTargets().length > 0) {
+ selectClause.append(generateTargetList(scan.getTargets()));
+ } else {
+ selectClause.append("1");
+ }
+
+ selectClause.append(" ");
+
+ ctx.sb.append("FROM ").append(scan.getTableName()).append(" ");
+
+ if (scan.hasAlias()) {
+ ctx.sb.append("AS ").append(scan.getAlias()).append(" ");
+ }
+
+ if (scan.hasQual()) {
+ ctx.sb.append("WHERE " + sqlExprGen.generate(scan.getQual()));
+ }
+ }
+
+ public String generateTargetList(Target [] targets) {
+ return StringUtils.join(targets, ",", new Function<Target, String>() {
+ @Override
+ public String apply(@Nullable Target t) {
+ StringBuilder sb = new StringBuilder(sqlExprGen.generate(t.getEvalTree()));
+ if (t.hasAlias()) {
+ sb.append(" AS ").append(t.getAlias());
+ }
+ return sb.toString();
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java
new file mode 100644
index 0000000..4d4c781
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java
@@ -0,0 +1,302 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.common.base.Function;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.common.TajoDataTypes.DataType;
+import org.apache.tajo.datum.BooleanDatum;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.exception.NotImplementedException;
+import org.apache.tajo.exception.TajoRuntimeException;
+import org.apache.tajo.exception.UnsupportedDataTypeException;
+import org.apache.tajo.plan.expr.*;
+import org.apache.tajo.util.StringUtils;
+
+import java.sql.DatabaseMetaData;
+import java.sql.SQLException;
+import java.util.Stack;
+
+/**
+ * A generator to build a SQL representation from a sql expression
+ */
+public class SQLExpressionGenerator extends SimpleEvalNodeVisitor<SQLExpressionGenerator.Context> {
+ final private DatabaseMetaData dbMetaData;
+
+ private final String LITERAL_QUOTE = "'";
+ @SuppressWarnings("unused")
+ private final String DEFAULT_LITERAL_QUOTE = "'";
+ @SuppressWarnings("unused")
+ private String IDENTIFIER_QUOTE = "\"";
+ private final String DEFAULT_IDENTIFIER_QUOTE = "\"";
+
+ public SQLExpressionGenerator(DatabaseMetaData dbMetaData) {
+ this.dbMetaData = dbMetaData;
+ initDatabaseDependentSQLRepr();
+ }
+
+ private void initDatabaseDependentSQLRepr() {
+ String quoteStr = null;
+ try {
+ quoteStr = dbMetaData.getIdentifierQuoteString();
+ } catch (SQLException e) {
+ }
+ this.IDENTIFIER_QUOTE = quoteStr != null ? quoteStr : DEFAULT_IDENTIFIER_QUOTE;
+ }
+
+ public String quote(String text) {
+ return LITERAL_QUOTE + text + LITERAL_QUOTE;
+ }
+
+ public String generate(EvalNode node) {
+ Context context = new Context();
+ visit(context, node, new Stack<EvalNode>());
+ return context.sb.toString();
+ }
+
+ public static class Context {
+ StringBuilder sb = new StringBuilder();
+
+ public void append(String text) {
+ sb.append(text).append(" ");
+ }
+ }
+
+ @Override
+ protected EvalNode visitUnaryEval(Context context, UnaryEval unary, Stack<EvalNode> stack) {
+
+ switch (unary.getType()) {
+ case NOT:
+ context.sb.append("NOT ");
+ super.visitUnaryEval(context, unary, stack);
+ break;
+ case SIGNED:
+ SignedEval signed = (SignedEval) unary;
+ if (signed.isNegative()) {
+ context.sb.append("-");
+ }
+ super.visitUnaryEval(context, unary, stack);
+ break;
+ case IS_NULL:
+ super.visitUnaryEval(context, unary, stack);
+
+ IsNullEval isNull = (IsNullEval) unary;
+ if (isNull.isNot()) {
+ context.sb.append("IS NOT NULL ");
+ } else {
+ context.sb.append("IS NULL ");
+ }
+ break;
+
+ case CAST:
+ super.visitUnaryEval(context, unary, stack);
+ context.sb.append(" AS ").append(convertTajoTypeToSQLType(unary.getValueType()));
+ }
+ return unary;
+ }
+
+ @Override
+ protected EvalNode visitBinaryEval(Context context, Stack<EvalNode> stack, BinaryEval binaryEval) {
+ stack.push(binaryEval);
+ visit(context, binaryEval.getLeftExpr(), stack);
+ context.sb.append(convertBinOperatorToSQLRepr(binaryEval.getType())).append(" ");
+ visit(context, binaryEval.getRightExpr(), stack);
+ stack.pop();
+ return binaryEval;
+ }
+
+ @Override
+ protected EvalNode visitConst(Context context, ConstEval constant, Stack<EvalNode> stack) {
+ context.sb.append(convertDatumToSQLLiteral(constant.getValue())).append(" ");
+ return constant;
+ }
+
+ protected EvalNode visitRowConstant(Context context, RowConstantEval row, Stack<EvalNode> stack) {
+ StringBuilder sb = new StringBuilder("(");
+ sb.append(StringUtils.join(row.getValues(), ",", new Function<Datum, String>() {
+ @Override
+ public String apply(Datum v) {
+ return convertDatumToSQLLiteral(v);
+ }
+ }));
+ sb.append(")");
+ context.append(sb.toString());
+
+ return row;
+ }
+
+ @Override
+ protected EvalNode visitField(Context context, FieldEval field, Stack<EvalNode> stack) {
+ // strip the database name
+ String tableName;
+ if (CatalogUtil.isSimpleIdentifier(field.getQualifier())) {
+ tableName = field.getQualifier();
+ } else {
+ tableName = CatalogUtil.extractSimpleName(field.getQualifier());
+ }
+
+ context.append(CatalogUtil.buildFQName(tableName, field.getColumnName()));
+ return field;
+ }
+
+ @Override
+ protected EvalNode visitBetween(Context context, BetweenPredicateEval evalNode, Stack<EvalNode> stack) {
+ stack.push(evalNode);
+ visit(context, evalNode.getPredicand(), stack);
+ context.append("BETWEEN");
+ visit(context, evalNode.getBegin(), stack);
+ context.append("AND");
+ visit(context, evalNode.getEnd(), stack);
+ return evalNode;
+ }
+
+ @Override
+ protected EvalNode visitCaseWhen(Context context, CaseWhenEval evalNode, Stack<EvalNode> stack) {
+ stack.push(evalNode);
+ context.append("CASE");
+ for (CaseWhenEval.IfThenEval ifThenEval : evalNode.getIfThenEvals()) {
+ visitIfThen(context, ifThenEval, stack);
+ }
+
+ context.append("ELSE");
+ if (evalNode.hasElse()) {
+ visit(context, evalNode.getElse(), stack);
+ }
+ stack.pop();
+ context.append("END");
+ return evalNode;
+ }
+
+ @Override
+ protected EvalNode visitIfThen(Context context, CaseWhenEval.IfThenEval evalNode, Stack<EvalNode> stack) {
+ stack.push(evalNode);
+ context.append("WHEN");
+ visit(context, evalNode.getCondition(), stack);
+ context.append("THEN");
+ visit(context, evalNode.getResult(), stack);
+ stack.pop();
+ return evalNode;
+ }
+
+ @Override
+ protected EvalNode visitFuncCall(Context context, FunctionEval func, Stack<EvalNode> stack) {
+ // TODO - TAJO-1837 should be resolved if we support RDBMS functions better.
+
+ stack.push(func);
+
+ context.sb.append(func.getName()).append("(");
+
+ boolean first = true;
+ for (EvalNode param : func.getArgs()) {
+ if (first) {
+ first = false;
+ } else {
+ context.sb.append(",");
+ }
+
+ visit(context, param, stack);
+ }
+
+ context.sb.append(")");
+ stack.pop();
+
+ return func;
+ }
+
+ @Override
+ protected EvalNode visitSubquery(Context context, SubqueryEval evalNode, Stack<EvalNode> stack) {
+ throw new TajoRuntimeException(new NotImplementedException());
+ }
+
+ /**
+ * convert Tajo literal into SQL representation
+ *
+ * @param d Datum
+ */
+ public String convertDatumToSQLLiteral(Datum d) {
+ switch (d.type()) {
+ case BOOLEAN:
+ return d.asBool() ? "TRUE" : "FALSE";
+
+ case INT1:
+ case INT2:
+ case INT4:
+ case INT8:
+ case FLOAT4:
+ case FLOAT8:
+ case NUMERIC:
+ return d.asChars();
+
+ case TEXT:
+ case VARCHAR:
+ case CHAR:
+ return quote(d.asChars());
+
+ case DATE:
+ return "DATE " + quote(d.asChars());
+
+ case TIME:
+ return "TIME " + quote(d.asChars());
+
+ case TIMESTAMP:
+ return "TIMESTAMP " + quote(d.asChars());
+
+ case NULL_TYPE:
+ return "NULL";
+
+ default:
+ throw new TajoRuntimeException(new UnsupportedDataTypeException(d.type().name()));
+ }
+ }
+
+ /**
+ * Convert Tajo DataType into SQL DataType
+ *
+ * @param dataType Tajo DataType
+ * @return SQL DataType
+ */
+ public String convertTajoTypeToSQLType(DataType dataType) {
+ switch (dataType.getType()) {
+ case INT1:
+ return "TINYINT";
+ case INT2:
+ return "SMALLINT";
+ case INT4:
+ return "INTEGER";
+ case INT8:
+ return "BIGINT";
+ case FLOAT4:
+ return "FLOAT";
+ case FLOAT8:
+ return "DOUBLE";
+ default:
+ return dataType.getType().name();
+ }
+ }
+
+ /**
+ * Convert EvalType the operator notation into SQL notation
+ *
+ * @param op EvalType
+ * @return SQL representation
+ */
+ public String convertBinOperatorToSQLRepr(EvalType op) {
+ return op.getOperatorName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto b/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto
new file mode 100644
index 0000000..f642e07
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/proto/JdbcFragmentProtos.proto
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tajo.storage.jdbc";
+option java_outer_classname = "JdbcFragmentProtos";
+option optimize_for = SPEED;
+option java_generic_services = false;
+option java_generate_equals_and_hash = true;
+
+import "CatalogProtos.proto";
+
+message JdbcFragmentProto {
+ required string uri = 1;
+ required string input_source_id = 2;
+ repeated string hosts = 3;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java b/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java
new file mode 100644
index 0000000..a04209f
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/test/java/org/apache/tajo/storage/jdbc/TestConnectionInfo.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestConnectionInfo {
+ @Test
+ public final void testGetConnectionInfoType1() {
+ ConnectionInfo c1 = ConnectionInfo.fromURI("jdbc:mysql://localhost:55840?user=testuser&password=testpass");
+ assertEquals("jdbc:mysql", c1.scheme);
+ assertEquals("localhost", c1.host);
+ assertEquals("testuser", c1.user);
+ assertEquals("testpass", c1.password);
+ assertNull(c1.dbName);
+ assertNull(c1.tableName);
+ assertEquals(0, c1.params.size());
+ }
+
+ @Test
+ public final void testGetConnectionInfoType2() {
+ ConnectionInfo c1 = ConnectionInfo.fromURI(
+ "jdbc:mysql://localhost:55840/db1?table=tb1&user=testuser&password=testpass&TZ=GMT+9");
+ assertEquals("jdbc:mysql", c1.scheme);
+ assertEquals("localhost", c1.host);
+ assertEquals("testuser", c1.user);
+ assertEquals("testpass", c1.password);
+ assertEquals("db1", c1.dbName);
+ assertEquals("tb1", c1.tableName);
+ assertEquals(1, c1.params.size());
+ assertEquals("GMT+9", c1.params.get("TZ"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml
new file mode 100644
index 0000000..c36d9bf
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/pom.xml
@@ -0,0 +1,257 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>tajo-project</artifactId>
+ <groupId>org.apache.tajo</groupId>
+ <version>0.12.0-SNAPSHOT</version>
+ <relativePath>../../tajo-project</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>tajo-storage-pgsql</artifactId>
+ <packaging>jar</packaging>
+ <name>Tajo PostgreSQL JDBC storage</name>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ <encoding>${project.build.sourceEncoding}</encoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>derby.log</exclude>
+ <exclude>src/test/resources/dataset/**</exclude>
+ <exclude>src/test/resources/queries/**</exclude>
+ <exclude>src/test/resources/results/**</exclude>
+ <exclude>src/test/resources/pgsql/**</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemProperties>
+ <tajo.test>TRUE</tajo.test>
+ </systemProperties>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-catalog-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-plan</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-jdbc</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-cluster-tests</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>testing-postgresql-server</artifactId>
+ <version>0.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>15.0</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>docs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <!-- build javadoc jars per jar for publishing to maven -->
+ <id>module-javadocs</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <destDir>${project.build.directory}</destDir>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <version>2.15</version>
+ </plugin>
+ </plugins>
+ </reporting>
+</project>
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLJdbcScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLJdbcScanner.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLJdbcScanner.java
new file mode 100644
index 0000000..2fa9f62
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLJdbcScanner.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.storage.jdbc.JdbcFragment;
+import org.apache.tajo.storage.jdbc.JdbcScanner;
+
+import java.sql.DatabaseMetaData;
+import java.util.Properties;
+
+public class PgSQLJdbcScanner extends JdbcScanner {
+
+ public PgSQLJdbcScanner(DatabaseMetaData dbMetaData,
+ Properties connProperties,
+ Schema tableSchema,
+ TableMeta tableMeta,
+ JdbcFragment fragment) {
+ super(dbMetaData, connProperties, tableSchema, tableMeta, fragment);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java
new file mode 100644
index 0000000..069ffe3
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLMetadataProvider.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import org.apache.tajo.storage.jdbc.JdbcMetadataProviderBase;
+
+import javax.annotation.Nullable;
+import java.util.Collection;
+
+public class PgSQLMetadataProvider extends JdbcMetadataProviderBase {
+
+ public PgSQLMetadataProvider(PgSQLTablespace space, String dbName) {
+ super(space, dbName);
+ }
+
+ @Override
+ protected String getJdbcDriverName() {
+ return "org.postgresql.Driver";
+ }
+
+ @Override
+ public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) {
+ return super.getTables("public", tablePattern);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java
new file mode 100644
index 0000000..1969d13
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import net.minidev.json.JSONObject;
+import org.apache.tajo.catalog.MetadataProvider;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.storage.NullScanner;
+import org.apache.tajo.storage.Scanner;
+import org.apache.tajo.storage.fragment.Fragment;
+import org.apache.tajo.storage.jdbc.JdbcFragment;
+import org.apache.tajo.storage.jdbc.JdbcTablespace;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * Postgresql Database Tablespace
+ */
+public class PgSQLTablespace extends JdbcTablespace {
+
+ public PgSQLTablespace(String name, URI uri, JSONObject config) {
+ super(name, uri, config);
+ }
+
+ public MetadataProvider getMetadataProvider() {
+ return new PgSQLMetadataProvider(this, database);
+ }
+
+ @Override
+ public Scanner getScanner(TableMeta meta,
+ Schema schema,
+ Fragment fragment,
+ @Nullable Schema target) throws IOException {
+ if (!(fragment instanceof JdbcFragment)) {
+ throw new TajoInternalError("fragment must be JdbcFragment");
+ }
+
+ if (target == null) {
+ target = schema;
+ }
+
+ Scanner scanner;
+ if (fragment.isEmpty()) {
+ scanner = new NullScanner(conf, schema, meta, fragment);
+ } else {
+ scanner = new PgSQLJdbcScanner(getDatabaseMetaData(), connProperties, schema, meta, (JdbcFragment) fragment);
+ }
+ scanner.setTarget(target.toArray());
+ return scanner;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java
new file mode 100644
index 0000000..024b497
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.pgsql;
+
+import com.google.common.base.Optional;
+import io.airlift.testing.postgresql.TestingPostgreSqlServer;
+import net.minidev.json.JSONObject;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.Tablespace;
+import org.apache.tajo.storage.TablespaceManager;
+import org.apache.tajo.storage.jdbc.JdbcTablespace;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
+
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Map;
+
+public class PgSQLTestServer {
+ private static final Log LOG = LogFactory.getLog(PgSQLTestServer.class);
+
+ private static PgSQLTestServer instance;
+
+ public static final String [] TPCH_TABLES = {
+ "customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier"
+ };
+
+ public static final String SPACENAME = "pgsql_cluster";
+ public static final String DATABASE_NAME = "tpch";
+ public static final String USERNAME = "testuser";
+ public static final String PASSWORD = "";
+
+ private final TestingPostgreSqlServer server;
+
+ static {
+ try {
+ instance = new PgSQLTestServer();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static PgSQLTestServer getInstance() {
+ return instance;
+ }
+
+ private PgSQLTestServer() throws Exception {
+ server = new TestingPostgreSqlServer(USERNAME,
+ "tpch"
+ );
+
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ server.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }));
+
+ loadTPCHTables();
+ registerTablespace();
+ }
+
+ Path testPath = CommonTestingUtil.getTestDir();
+
+ private void loadTPCHTables() throws SQLException, IOException {
+
+
+ try (Connection connection = DriverManager.getConnection(getJdbcUrlForAdmin(), "postgres", null)) {
+ connection.setCatalog("tpch");
+
+ try (Statement statement = connection.createStatement()) {
+
+ for (String tableName : TPCH_TABLES) {
+ String sql = JavaResourceUtil.readTextFromResource("pgsql/" + tableName + ".sql");
+ statement.executeUpdate(sql);
+
+ // restore the table contents into a file stored in a local file system for PgSQL COPY command
+ String path = restoreTableContents(tableName);
+ String copyCommand = genLoadStatement(tableName, path);
+ statement.executeUpdate(copyCommand);
+ }
+
+ // load DATETIME_TYPES table
+ String sql = JavaResourceUtil.readTextFromResource("pgsql/datetime_types.sql");
+ statement.executeUpdate(sql);
+ Path filePath = new Path(testPath, "datetime_types.txt");
+ storeTableContents("pgsql/datetime_types.txt", filePath);
+ String copyCommand = genLoadStatement("datetime_types", filePath.toUri().getPath());
+ LOG.info(copyCommand);
+ statement.executeUpdate(copyCommand);
+
+ } catch (Throwable t) {
+ t.printStackTrace();
+ throw t;
+ }
+ }
+ }
+
+ private String genLoadStatement(String tableName, String path) {
+ return "COPY " + tableName + " FROM '" + path + "' WITH (FORMAT csv, DELIMITER '|');";
+ }
+
+ private void storeTableContents(String resource, Path path) throws IOException {
+ String csvTable = JavaResourceUtil.readTextFromResource(resource);
+ String fixedCsvTable = fixExtraColumn(csvTable);
+ FileUtil.writeTextToFile(fixedCsvTable, path);
+ }
+
+ private String restoreTableContents(String tableName) throws IOException {
+ Path filePath = new Path(testPath, tableName + ".tbl");
+ storeTableContents("tpch/" + tableName + ".tbl", filePath);
+ return filePath.toUri().getPath();
+ }
+
+ private String fixExtraColumn(String csvTable) {
+ final String [] lines = csvTable.split("\n");
+ final StringBuilder rewritten = new StringBuilder();
+
+ for (String l : lines) {
+ if (l.charAt(l.length() - 1) == '|') {
+ rewritten.append(l.substring(0, l.length() - 1));
+ } else {
+ rewritten.append(l.substring(0, l.length()));
+ }
+ rewritten.append("\n");
+ }
+
+ return rewritten.toString();
+ }
+
+ private void registerTablespace() throws IOException {
+ JSONObject configElements = new JSONObject();
+ configElements.put(JdbcTablespace.CONFIG_KEY_MAPPED_DATABASE, DATABASE_NAME);
+
+ PgSQLTablespace tablespace = new PgSQLTablespace(SPACENAME, URI.create(getJdbcUrlForAdmin()), configElements);
+ tablespace.init(new TajoConf());
+
+ TablespaceManager.addTableSpaceForTest(tablespace);
+ }
+
+ /**
+ * get JDBC URL for a created user
+ *
+ * @return JDBC URL for the created user
+ */
+ public String getJdbcUrl() {
+ return server.getJdbcUrl() + "&connectTimeout=5&socketTimeout=5";
+ }
+
+ /**
+ * get JDBC URL for the Admin user
+ *
+ * @return JDBC URL for the Admin user
+ */
+ public String getJdbcUrlForAdmin() {
+ // replace 'user' by postgres (admin)
+ String url = server.getJdbcUrl().split("\\?")[0];
+ url += "?user=postgres&connectTimeout=5&socketTimeout=5";
+ return url;
+ }
+
+ public TestingPostgreSqlServer getServer() {
+ return server;
+ }
+
+ public static Optional<Tablespace> resetAllParamsAndSetConnProperties(Map<String, String> connProperties)
+ throws IOException {
+ String uri = PgSQLTestServer.getInstance().getJdbcUrl().split("\\?")[0];
+
+ JSONObject configElements = new JSONObject();
+ configElements.put(JdbcTablespace.CONFIG_KEY_MAPPED_DATABASE, PgSQLTestServer.DATABASE_NAME);
+
+ JSONObject connPropertiesJson = new JSONObject();
+ for (Map.Entry<String, String> entry : connProperties.entrySet()) {
+ connPropertiesJson.put(entry.getKey(), entry.getValue());
+ }
+ configElements.put(JdbcTablespace.CONFIG_KEY_CONN_PROPERTIES, connPropertiesJson);
+
+ PgSQLTablespace tablespace = new PgSQLTablespace(PgSQLTestServer.SPACENAME, URI.create(uri), configElements);
+ tablespace.init(new TajoConf());
+ return TablespaceManager.addTableSpaceForTest(tablespace);
+ }
+}
[3/4] tajo git commit: TAJO-1730: JDBC Tablespace support.
Posted by hy...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 4e74862..504c625 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -1339,7 +1339,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
QueryBlock block = context.queryBlock;
ScanNode scanNode = block.getNodeFromExpr(expr);
- updatePhysicalInfo(context, scanNode.getTableDesc());
+ updatePhysicalInfo(scanNode.getTableDesc());
// Find expression which can be evaluated at this relation node.
// Except for column references, additional expressions used in select list, where clause, order-by clauses
@@ -1398,24 +1398,18 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return targets;
}
- private void updatePhysicalInfo(PlanContext planContext, TableDesc desc) {
- if (desc.getUri() != null &&
- !desc.getMeta().getStoreType().equals("SYSTEM") &&
- !desc.getMeta().getStoreType().equals("FAKEFILE") && // FAKEFILE is used for test
- PlannerUtil.isFileStorageType(desc.getMeta().getStoreType())) {
+ private void updatePhysicalInfo(TableDesc desc) {
+
+ // FAKEFILE is used for test
+ if (!desc.getMeta().getStoreType().equals("SYSTEM") && !desc.getMeta().getStoreType().equals("FAKEFILE")) {
try {
- Path path = new Path(desc.getUri());
- FileSystem fs = path.getFileSystem(planContext.queryContext.getConf());
- FileStatus status = fs.getFileStatus(path);
- if (desc.getStats() != null && (status.isDirectory() || status.isFile())) {
- ContentSummary summary = fs.getContentSummary(path);
- if (summary != null) {
- long volume = summary.getLength();
- desc.getStats().setNumBytes(volume);
- }
+ if (desc.getStats() != null) {
+ desc.getStats().setNumBytes(storage.getTableVolumn(desc.getUri()));
}
- } catch (Throwable t) {
- LOG.warn(t, t);
+ } catch (UnsupportedException t) {
+ LOG.warn(desc.getName() + " does not support Tablespace::getTableVolume()");
+ // -1 means unknown volume size.
+ desc.getStats().setNumBytes(-1);
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
index d06c1d3..19d5d16 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
@@ -156,7 +156,7 @@ public class AlgebraicUtil {
}
@Override
- public EvalNode visitUnaryEval(Object context, Stack<EvalNode> stack, UnaryEval unaryEval) {
+ public EvalNode visitUnaryEval(Object context, UnaryEval unaryEval, Stack<EvalNode> stack) {
stack.push(unaryEval);
EvalNode child = visit(context, unaryEval.getChild(), stack);
stack.pop();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java
index 2c2a52f..35390a5 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java
@@ -21,10 +21,11 @@ package org.apache.tajo.plan.expr;
public enum EvalType {
// Unary expression
NOT(NotEval.class, "!"),
+ SIGNED(SignedEval.class),
// Binary expression
- AND(BinaryEval.class),
- OR(BinaryEval.class),
+ AND(BinaryEval.class, "AND"),
+ OR(BinaryEval.class, "OR"),
EQUAL(BinaryEval.class, "="),
IS_NULL(IsNullEval.class),
NOT_EQUAL(BinaryEval.class, "<>"),
@@ -49,19 +50,18 @@ public enum EvalType {
FUNCTION(GeneralFunctionEval.class),
// String operator or pattern matching predicates
- LIKE(LikePredicateEval.class),
- SIMILAR_TO(SimilarToPredicateEval.class),
- REGEX(RegexPredicateEval.class),
+ LIKE(LikePredicateEval.class, "LIKE"),
+ SIMILAR_TO(SimilarToPredicateEval.class, "SIMILAR TO"),
+ REGEX(RegexPredicateEval.class, "REGEX"),
CONCATENATE(BinaryEval.class, "||"),
// Other predicates
BETWEEN(BetweenPredicateEval.class),
CASE(CaseWhenEval.class),
IF_THEN(CaseWhenEval.IfThenEval.class),
- IN(InEval.class),
+ IN(InEval.class, "IN"),
// Value or Reference
- SIGNED(SignedEval.class),
CAST(CastEval.class),
ROW_CONSTANT(RowConstantEval.class),
FIELD(FieldEval.class),
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java
index 9515fe8..abaf5fe 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java
@@ -20,7 +20,6 @@ package org.apache.tajo.plan.expr;
import com.google.common.base.Preconditions;
import org.apache.tajo.exception.TajoInternalError;
-import org.apache.tajo.exception.UnsupportedException;
import java.util.Stack;
@@ -36,7 +35,7 @@ public abstract class SimpleEvalNodeVisitor<CONTEXT> {
EvalNode result;
if (evalNode instanceof UnaryEval) {
- result = visitUnaryEval(context, stack, (UnaryEval) evalNode);
+ result = visitUnaryEval(context, (UnaryEval) evalNode, stack);
} else if (evalNode instanceof BinaryEval) {
result = visitBinaryEval(context, stack, (BinaryEval) evalNode);
} else {
@@ -50,7 +49,7 @@ public abstract class SimpleEvalNodeVisitor<CONTEXT> {
result = visitRowConstant(context, (RowConstantEval) evalNode, stack);
break;
case FIELD:
- result = visitField(context, stack, (FieldEval) evalNode);
+ result = visitField(context, (FieldEval) evalNode, stack);
break;
@@ -88,7 +87,7 @@ public abstract class SimpleEvalNodeVisitor<CONTEXT> {
return result;
}
- protected EvalNode visitUnaryEval(CONTEXT context, Stack<EvalNode> stack, UnaryEval unaryEval) {
+ protected EvalNode visitUnaryEval(CONTEXT context, UnaryEval unaryEval, Stack<EvalNode> stack) {
stack.push(unaryEval);
visit(context, unaryEval.getChild(), stack);
stack.pop();
@@ -126,7 +125,7 @@ public abstract class SimpleEvalNodeVisitor<CONTEXT> {
return evalNode;
}
- protected EvalNode visitField(CONTEXT context, Stack<EvalNode> stack, FieldEval evalNode) {
+ protected EvalNode visitField(CONTEXT context, FieldEval evalNode, Stack<EvalNode> stack) {
return evalNode;
}
@@ -163,10 +162,6 @@ public abstract class SimpleEvalNodeVisitor<CONTEXT> {
return evalNode;
}
- protected EvalNode visitInPredicate(CONTEXT context, InEval evalNode, Stack<EvalNode> stack) {
- return visitBinaryEval(context, stack, evalNode);
- }
-
///////////////////////////////////////////////////////////////////////////////////////////////
// Functions
///////////////////////////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
index 95f4c76..37b77fd 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
@@ -64,7 +64,7 @@ public class ConstantFolding extends SimpleEvalNodeVisitor<LogicalPlanner.PlanCo
}
@Override
- public EvalNode visitUnaryEval(LogicalPlanner.PlanContext context, Stack<EvalNode> stack, UnaryEval unaryEval) {
+ public EvalNode visitUnaryEval(LogicalPlanner.PlanContext context, UnaryEval unaryEval, Stack<EvalNode> stack) {
stack.push(unaryEval);
EvalNode child = visit(context, unaryEval.getChild(), stack);
stack.pop();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java
index 6fe3b3e..24488bd 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java
@@ -72,7 +72,7 @@ public class ConstantPropagation extends SimpleEvalNodeVisitor<LogicalPlanner.Pl
}
@Override
- public EvalNode visitUnaryEval(LogicalPlanner.PlanContext context, Stack<EvalNode> stack, UnaryEval unaryEval) {
+ public EvalNode visitUnaryEval(LogicalPlanner.PlanContext context, UnaryEval unaryEval, Stack<EvalNode> stack) {
stack.push(unaryEval);
if (unaryEval.getChild().getType() == EvalType.FIELD) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java
index 8e369d6..4b8a9e9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java
@@ -19,6 +19,7 @@
package org.apache.tajo.plan.logical;
import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
@@ -37,6 +38,7 @@ public class ScanNode extends RelationNode implements Projectable, SelectableNod
@Expose protected EvalNode qual;
@Expose protected Target[] targets;
@Expose protected boolean broadcastTable;
+ @Expose protected long limit = -1; // -1 means no set
protected ScanNode(int pid, NodeType nodeType) {
super(pid, nodeType);
@@ -150,6 +152,29 @@ public class ScanNode extends RelationNode implements Projectable, SelectableNod
return this.targets;
}
+ /**
+ *
+ *
+ * @return
+ */
+ public boolean hasLimit() {
+ return limit > 0;
+ }
+
+ /**
+ * How many rows will be retrieved?
+ *
+ * @return The number of rows to be retrieved
+ */
+ public long getLimit() {
+ return limit;
+ }
+
+ public void setLimit(long num) {
+ Preconditions.checkArgument(num > 0, "The number of fetch rows in limit is negative");
+ this.limit = num;
+ }
+
public TableDesc getTableDesc() {
return tableDesc;
}
@@ -249,9 +274,4 @@ public class ScanNode extends RelationNode implements Projectable, SelectableNod
return planStr;
}
-
- public static boolean isScanNode(LogicalNode node) {
- return node.getType() == NodeType.SCAN ||
- node.getType() == NodeType.PARTITIONS_SCAN;
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java
index f66350a..5524256 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java
@@ -111,7 +111,7 @@ public class CommonConditionReduceRule implements LogicalPlanRewriteRule {
}
@Override
- protected EvalNode visitUnaryEval(Object context, Stack<EvalNode> stack, UnaryEval unaryEval) {
+ protected EvalNode visitUnaryEval(Object context, UnaryEval unaryEval, Stack<EvalNode> stack) {
stack.push(unaryEval);
EvalNode child = unaryEval.getChild();
visit(context, child, stack);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java
index 7de0b05..fef1528 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java
@@ -102,9 +102,9 @@ public class EvalNodeSerializer
}
@Override
- public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack<EvalNode> stack, UnaryEval unary) {
+ public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, UnaryEval unary, Stack<EvalNode> stack) {
// visiting and registering childs
- super.visitUnaryEval(context, stack, unary);
+ super.visitUnaryEval(context, unary, stack);
int [] childIds = registerGetChildIds(context, unary);
// building itself
@@ -183,7 +183,7 @@ public class EvalNodeSerializer
}
@Override
- public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack<EvalNode> stack, FieldEval field) {
+ public EvalNode visitField(EvalTreeProtoBuilderContext context, FieldEval field, Stack<EvalNode> stack) {
PlanProto.EvalNode.Builder builder = createEvalBuilder(context, field);
builder.setField(field.getColumnRef().getProto());
context.treeBuilder.addNodes(builder);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index d9fb218..0210865 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -560,20 +560,6 @@ public class PlannerUtil {
}
}
- /**
- * fill targets with FieldEvals from a given schema
- *
- * @param schema to be transformed to targets
- * @param targets to be filled
- */
- public static void schemaToTargets(Schema schema, Target[] targets) {
- FieldEval eval;
- for (int i = 0; i < schema.size(); i++) {
- eval = new FieldEval(schema.getColumn(i));
- targets[i] = new Target(eval);
- }
- }
-
public static Target[] schemaToTargets(Schema schema) {
Target[] targets = new Target[schema.size()];
@@ -585,17 +571,6 @@ public class PlannerUtil {
return targets;
}
- public static Target[] schemaToTargetsWithGeneratedFields(Schema schema) {
- List<Target> targets = TUtil.newList();
-
- FieldEval eval;
- for (int i = 0; i < schema.size(); i++) {
- eval = new FieldEval(schema.getColumn(i));
- targets.add(new Target(eval));
- }
- return targets.toArray(new Target[targets.size()]);
- }
-
public static SortSpec[] schemaToSortSpecs(Schema schema) {
return columnsToSortSpecs(schema.toArray());
}
@@ -993,6 +968,25 @@ public class PlannerUtil {
return inSubqueries;
}
+ /**
+ * Return a list of integers, maps input schema and projected columns.
+ * Each integer value means a column index of input schema corresponding to each project column
+ *
+ * @param inputSchema Input Schema
+ * @param targets Columns to be projected
+ * @return A list of integers, each of which is an index number of input schema corresponding
+ * to each projected column.
+ */
+ public static int [] getTargetIds(Schema inputSchema, Column...targets) {
+ int [] targetIds = new int[targets.length];
+ for (int i = 0; i < targetIds.length; i++) {
+ targetIds[i] = inputSchema.getColumnId(targets[i].getQualifiedName());
+ }
+ Arrays.sort(targetIds);
+
+ return targetIds;
+ }
+
public static List<EvalNode> getAllEqualEvals(EvalNode qual) {
EvalTreeUtil.EvalFinder finder = new EvalTreeUtil.EvalFinder(EvalType.EQUAL);
finder.visit(null, qual, new Stack<EvalNode>());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-plan/src/main/proto/Plan.proto
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto
index 0bfac0d..7815cac 100644
--- a/tajo-plan/src/main/proto/Plan.proto
+++ b/tajo-plan/src/main/proto/Plan.proto
@@ -577,10 +577,10 @@ message EnforceProperty {
SORTED_INPUT = 0;
OUTPUT_DISTINCT = 1;
GROUP_BY = 2;
- JOIN = 3;
- SORT = 4;
- BROADCAST = 5;
- COLUMN_PARTITION = 6;
+ JOIN = 3;
+ SORT = 4;
+ BROADCAST = 5;
+ COLUMN_PARTITION = 6;
DISTINCT_GROUP_BY = 7;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index 9cddbf6..ee27fe3 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -786,6 +786,17 @@
</dependency>
<dependency>
<groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-jdbc</artifactId>
+ <version>${tajo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-jdbc</artifactId>
+ <version>${tajo.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
<artifactId>tajo-pullserver</artifactId>
<version>${tajo.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml
index 8f7b195..24dff10 100644
--- a/tajo-storage/pom.xml
+++ b/tajo-storage/pom.xml
@@ -37,6 +37,8 @@
<module>tajo-storage-common</module>
<module>tajo-storage-hdfs</module>
<module>tajo-storage-hbase</module>
+ <module>tajo-storage-jdbc</module>
+ <module>tajo-storage-pgsql</module>
</modules>
<build>
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/AbstractScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/AbstractScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/AbstractScanner.java
index c3dc959..cc8ad83 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/AbstractScanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/AbstractScanner.java
@@ -25,6 +25,7 @@ import org.apache.tajo.exception.NotImplementedException;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.LogicalNode;
import java.io.IOException;
@@ -49,6 +50,11 @@ public abstract class AbstractScanner implements Scanner {
}
@Override
+ public void pushOperators(LogicalNode planPart) {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ @Override
public boolean isProjectable() {
throw new TajoRuntimeException(new NotImplementedException());
}
@@ -69,6 +75,10 @@ public abstract class AbstractScanner implements Scanner {
}
@Override
+ public void setLimit(long num) {
+ }
+
+ @Override
public boolean isSplittable() {
throw new TajoRuntimeException(new NotImplementedException());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
index db64992..b3b1edd 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
@@ -28,6 +28,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.storage.fragment.Fragment;
import java.io.IOException;
@@ -153,6 +154,11 @@ public class MergeScanner implements Scanner {
}
@Override
+ public void pushOperators(LogicalNode planPart) {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ @Override
public boolean isProjectable() {
return projectable;
}
@@ -173,6 +179,10 @@ public class MergeScanner implements Scanner {
}
@Override
+ public void setLimit(long num) {
+ }
+
+ @Override
public Schema getSchema() {
return schema;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java
index 9c025f7..a20adf7 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/NullScanner.java
@@ -26,6 +26,7 @@ import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.storage.fragment.Fragment;
import java.io.IOException;
@@ -73,6 +74,11 @@ public class NullScanner implements Scanner {
}
@Override
+ public void pushOperators(LogicalNode planPart) {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ @Override
public boolean isProjectable() {
return false;
}
@@ -93,6 +99,11 @@ public class NullScanner implements Scanner {
}
@Override
+ public void setLimit(long num) {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ @Override
public boolean isSplittable() {
return true;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java
index d12c6bd..2e00bc9 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java
@@ -161,7 +161,7 @@ public class OldStorageManager {
constructor.setAccessible(true);
CONSTRUCTOR_CACHE.put(storageManagerClass, constructor);
}
- manager = constructor.newInstance(new Object[]{"noname", uri});
+ manager = constructor.newInstance(new Object[]{"noname", uri, null});
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
index 2fcb2fd..e95f318 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
@@ -22,6 +22,7 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.SchemaObject;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.LogicalNode;
import java.io.Closeable;
import java.io.IOException;
@@ -58,6 +59,12 @@ public interface Scanner extends SchemaObject, Closeable {
*/
void close() throws IOException;
+ /**
+ * Push a plan part into scanner. It will be used in future issues.
+ *
+ * @param planPart
+ */
+ void pushOperators(LogicalNode planPart);
/**
* It returns if the projection is executed in the underlying scanner layer.
@@ -96,6 +103,15 @@ public interface Scanner extends SchemaObject, Closeable {
*/
void setFilter(EvalNode filter);
+
+ /**
+ * This method does not guarantee that the scanner will retrieve the specified number of rows.
+ * This information is used for a hint.
+ *
+ * @param num The number of rows to be retrieved.
+ */
+ void setLimit(long num);
+
/**
* It returns if the file is splittable.
*
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
index c1db34e..41ecd38 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
@@ -30,16 +30,20 @@ public class StorageProperty {
private final boolean writable;
/** if this storage allows use of artibrary paths */
private final boolean absolutePathAllowed;
+ /** if this storage provides metadata provider */
+ private final boolean metadataProvided;
public StorageProperty(String defaultFormat,
boolean movable,
boolean writable,
- boolean absolutePathAllowed) {
+ boolean absolutePathAllowed,
+ boolean metadataProvided) {
this.defaultFormat = defaultFormat;
this.movable = movable;
this.writable = writable;
this.absolutePathAllowed = absolutePathAllowed;
+ this.metadataProvided = metadataProvided;
}
/**
@@ -76,4 +80,13 @@ public class StorageProperty {
public boolean isArbitraryPathAllowed() {
return this.absolutePathAllowed;
}
+
+ /**
+ * Is metadata provided?
+ *
+ * @return True if this storage provides linked metadata.
+ */
+ public boolean isMetadataProvided() {
+ return this.metadataProvided;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
index d131d9e..1c6f433 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
@@ -18,30 +18,28 @@
package org.apache.tajo.storage;
+import net.minidev.json.JSONObject;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.TajoRuntimeException;
import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.net.URI;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.Set;
/**
@@ -54,14 +52,20 @@ public abstract class Tablespace {
protected final String name;
protected final URI uri;
+ protected final JSONObject config;
/** this space is visible or not. */
protected boolean visible = true;
protected TajoConf conf;
- public Tablespace(String name, URI uri) {
+ public Tablespace(String name, URI uri, JSONObject config) {
this.name = name;
this.uri = uri;
+ this.config = config;
+ }
+
+ public JSONObject getConfig() {
+ return config;
}
public void setVisible(boolean visible) {
@@ -90,15 +94,11 @@ public abstract class Tablespace {
return visible;
}
- public abstract void setConfig(String name, String value);
-
- public abstract void setConfigs(Map<String, String> configs);
-
public String toString() {
return name + "=" + uri.toString();
}
- public abstract long getTableVolume(URI uri) throws IOException;
+ public abstract long getTableVolume(URI uri) throws UnsupportedException;
/**
* if {@link StorageProperty#isArbitraryPathAllowed} is true,
@@ -114,22 +114,24 @@ public abstract class Tablespace {
/**
* Get Table URI
*
- * @param tableName
- * @return
+ * @param databaseName Database name
+ * @param tableName Table name
+ * @return Table URI
*/
public abstract URI getTableUri(String databaseName, String tableName);
/**
* Returns the splits that will serve as input for the scan tasks. The
* number of splits matches the number of regions in a table.
- * @param fragmentId The table name or previous ExecutionBlockId
+ * @param inputSourceId Input source identifier, which can be either relation name or execution block id
* @param tableDesc The table description for the target data.
- * @param scanNode The logical node for scanning.
+ * @param filterCondition filter condition which can prune splits if possible
* @return The list of input fragments.
* @throws java.io.IOException
*/
- public abstract List<Fragment> getSplits(String fragmentId, TableDesc tableDesc,
- ScanNode scanNode) throws IOException, TajoException;
+ public abstract List<Fragment> getSplits(String inputSourceId,
+ TableDesc tableDesc,
+ @Nullable EvalNode filterCondition) throws IOException, TajoException;
/**
* It returns the storage property.
@@ -181,46 +183,6 @@ public abstract class Tablespace {
}
/**
- * Returns the splits that will serve as input for the scan tasks. The
- * number of splits matches the number of regions in a table.
- *
- * @param fragmentId The table name or previous ExecutionBlockId
- * @param tableDesc The table description for the target data.
- * @return The list of input fragments.
- * @throws java.io.IOException
- */
- public List<Fragment> getSplits(String fragmentId, TableDesc tableDesc) throws IOException, TajoException {
- return getSplits(fragmentId, tableDesc, null);
- }
-
- /**
- * Returns Scanner instance.
- *
- * @param meta The table meta
- * @param schema The input schema
- * @param fragment The fragment for scanning
- * @param target Columns which are selected.
- * @return Scanner instance
- * @throws java.io.IOException
- */
- public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException {
- return getScanner(meta, schema, FragmentConvertor.convert(conf, fragment), target);
- }
-
- /**
- * Returns Scanner instance.
- *
- * @param meta The table meta
- * @param schema The input schema
- * @param fragment The fragment for scanning
- * @return Scanner instance
- * @throws java.io.IOException
- */
- public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException {
- return getScanner(meta, schema, fragment, schema);
- }
-
- /**
* Returns Scanner instance.
*
* @param meta The table meta
@@ -230,7 +192,14 @@ public abstract class Tablespace {
* @return Scanner instance
* @throws java.io.IOException
*/
- public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException {
+ public Scanner getScanner(TableMeta meta,
+ Schema schema,
+ Fragment fragment,
+ @Nullable Schema target) throws IOException {
+ if (target == null) {
+ target = schema;
+ }
+
if (fragment.isEmpty()) {
Scanner scanner = new NullScanner(conf, schema, meta, fragment);
scanner.setTarget(target.toArray());
@@ -267,7 +236,7 @@ public abstract class Tablespace {
*/
public synchronized SeekableScanner getSeekableScanner(TableMeta meta, Schema schema, FragmentProto fragment,
Schema target) throws IOException {
- return (SeekableScanner)this.getScanner(meta, schema, fragment, target);
+ return (SeekableScanner)this.getScanner(meta, schema, FragmentConvertor.convert(conf, fragment), target);
}
/**
@@ -413,4 +382,13 @@ public abstract class Tablespace {
TableMeta meta) throws IOException {
throw new IOException("Staging the output result is not supported in this storage");
}
+
+ public MetadataProvider getMetadataProvider() {
+ throw new TajoRuntimeException(new UnsupportedException("Linked Metadata Provider for " + name));
+ }
+
+ @SuppressWarnings("unused")
+ public int markAccetablePlanPart(LogicalPlan plan) {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
index ef71ada..a1fa857 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
@@ -19,8 +19,10 @@
package org.apache.tajo.storage;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
import com.google.common.collect.Maps;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
@@ -29,16 +31,20 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.TajoConstants;
+import org.apache.tajo.catalog.MetadataProvider;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.storage.fragment.Fragment;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.UriUtil;
import javax.annotation.Nullable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URI;
+import java.util.Collection;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
@@ -71,11 +77,14 @@ public class TablespaceManager implements StorageService {
protected static final Map<Class<?>, Constructor<?>> CONSTRUCTORS = Maps.newHashMap();
protected static final Map<String, Class<? extends Tablespace>> TABLE_SPACE_HANDLERS = Maps.newHashMap();
- public static final Class [] TABLESPACE_PARAM = new Class [] {String.class, URI.class};
+ public static final Class[] TABLESPACE_PARAM = new Class[]{String.class, URI.class, JSONObject.class};
+
+ public static final String TABLESPACE_SPEC_CONFIGS_KEY = "configs";
static {
instance = new TablespaceManager();
}
+
/**
* Singleton instance
*/
@@ -94,7 +103,7 @@ public class TablespaceManager implements StorageService {
}
private void addLocalFsTablespace() {
- if (TABLE_SPACES.headMap(LOCAL_FS_URI, true).firstEntry() == null) {
+ if (TABLE_SPACES.headMap(LOCAL_FS_URI, true).firstEntry() == null && TABLE_SPACE_HANDLERS.containsKey("file")) {
String tmpName = UUID.randomUUID().toString();
registerTableSpace(tmpName, LOCAL_FS_URI, null, false, false);
}
@@ -124,9 +133,9 @@ public class TablespaceManager implements StorageService {
private JSONObject loadFromConfig(String fileName) {
String json;
try {
- json = FileUtil.readTextFileFromResource(fileName);
+ json = JavaResourceUtil.readTextFromResource(fileName);
} catch (FileNotFoundException fnfe) {
- return null;
+ return null;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -174,7 +183,7 @@ public class TablespaceManager implements StorageService {
String handlerClass = (String) storageDesc.get(KEY_STORAGE_HANDLER);
return new Pair<String, Class<? extends Tablespace>>(
- storageType,(Class<? extends Tablespace>) Class.forName(handlerClass));
+ storageType, (Class<? extends Tablespace>) Class.forName(handlerClass));
}
private void loadTableSpaces(JSONObject json, boolean override) {
@@ -182,24 +191,29 @@ public class TablespaceManager implements StorageService {
if (spaces != null) {
for (Map.Entry<String, Object> entry : spaces.entrySet()) {
- AddTableSpace(entry.getKey(), (JSONObject) entry.getValue(), override);
+ JSONObject spaceDetail = (JSONObject) entry.getValue();
+ AddTableSpace(
+ entry.getKey(),
+ URI.create(spaceDetail.getAsString("uri")),
+ Boolean.parseBoolean(spaceDetail.getAsString("default")),
+ (JSONObject) spaceDetail.get(TABLESPACE_SPEC_CONFIGS_KEY),
+ override);
}
}
}
- public static void AddTableSpace(String spaceName, JSONObject spaceDesc, boolean override) {
- boolean defaultSpace = Boolean.parseBoolean(spaceDesc.getAsString("default"));
- URI spaceUri = URI.create(spaceDesc.getAsString("uri"));
+ public static void AddTableSpace(String spaceName, URI uri, boolean isDefault, JSONObject configs, boolean override) {
- if (defaultSpace) {
- registerTableSpace(DEFAULT_TABLESPACE_NAME, spaceUri, spaceDesc, true, override);
+
+ if (isDefault) {
+ registerTableSpace(DEFAULT_TABLESPACE_NAME, uri, configs, true, override);
}
- registerTableSpace(spaceName, spaceUri, spaceDesc, true, override);
+ registerTableSpace(spaceName, uri, configs, true, override);
}
private static void registerTableSpace(String spaceName, URI uri, JSONObject spaceDesc,
boolean visible, boolean override) {
- Tablespace tableSpace = initializeTableSpace(spaceName, uri, visible);
+ Tablespace tableSpace = initializeTableSpace(spaceName, uri, spaceDesc);
tableSpace.setVisible(visible);
try {
@@ -261,12 +275,11 @@ public class TablespaceManager implements StorageService {
public static final String KEY_SPACES = "spaces";
- private static Tablespace initializeTableSpace(String spaceName, URI uri, boolean visible) {
- Preconditions.checkNotNull(uri.getScheme(), "URI must include scheme, but it was " + uri);
- Class<? extends Tablespace> clazz = TABLE_SPACE_HANDLERS.get(uri.getScheme());
+ private static Tablespace initializeTableSpace(String spaceName, URI uri, JSONObject spaceDesc) {
+ Class<? extends Tablespace> clazz = TABLE_SPACE_HANDLERS.get(UriUtil.getScheme(uri));
if (clazz == null) {
- throw new RuntimeException("There is no tablespace for " + uri.toString());
+ throw new RuntimeException("Not found Tablespace handler for " + uri.toString());
}
try {
@@ -279,7 +292,7 @@ public class TablespaceManager implements StorageService {
CONSTRUCTORS.put(clazz, constructor);
}
- return constructor.newInstance(new Object[]{spaceName, uri});
+ return constructor.newInstance(new Object[]{spaceName, uri, spaceDesc});
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -289,12 +302,18 @@ public class TablespaceManager implements StorageService {
public static Optional<Tablespace> addTableSpaceForTest(Tablespace space) {
Tablespace existing;
synchronized (SPACES_URIS_MAP) {
+
+ String scheme = UriUtil.getScheme(space.getUri());
+ if (!TABLE_SPACE_HANDLERS.containsKey(scheme)) {
+ TABLE_SPACE_HANDLERS.put(scheme, space.getClass());
+ }
+
// Remove existing one
SPACES_URIS_MAP.remove(space.getName());
existing = TABLE_SPACES.remove(space.getUri());
// Add anotherone for test
- registerTableSpace(space.name, space.uri, null, true, true);
+ registerTableSpace(space.name, space.uri, space.getConfig(), true, true);
}
// if there is an existing one, return it.
return Optional.fromNullable(existing);
@@ -321,7 +340,7 @@ public class TablespaceManager implements StorageService {
// Find the longest matched one. For example, assume that the caller tries to find /x/y/z, and
// there are /x and /x/y. In this case, /x/y will be chosen because it is more specific.
- for (Map.Entry<URI, Tablespace> entry: TABLE_SPACES.headMap(URI.create(uri), true).entrySet()) {
+ for (Map.Entry<URI, Tablespace> entry : TABLE_SPACES.headMap(URI.create(uri), true).entrySet()) {
if (uri.startsWith(entry.getKey().toString())) {
lastOne = entry.getValue();
}
@@ -383,7 +402,28 @@ public class TablespaceManager implements StorageService {
return space.getTableUri(databaseName, tableName);
}
+ @Override
+ public long getTableVolumn(URI tableUri) throws UnsupportedException {
+ return get(tableUri).get().getTableVolume(tableUri);
+ }
+
public static Iterable<Tablespace> getAllTablespaces() {
return TABLE_SPACES.values();
}
+
+ public static Collection<MetadataProvider> getMetadataProviders() {
+ Collection<Tablespace> filteredSpace = Collections2.filter(TABLE_SPACES.values(), new Predicate<Tablespace>() {
+ @Override
+ public boolean apply(@Nullable Tablespace space) {
+ return space.getProperty().isMetadataProvided();
+ }
+ });
+
+ return Collections2.transform(filteredSpace, new Function<Tablespace, MetadataProvider>() {
+ @Override
+ public MetadataProvider apply(@Nullable Tablespace space) {
+ return space.getMetadataProvider();
+ }
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
index 07720c7..bd46551 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Maps;
import com.google.protobuf.ByteString;
import org.apache.hadoop.conf.Configuration;
import org.apache.tajo.annotation.ThreadSafe;
+import org.apache.tajo.exception.TajoInternalError;
import java.io.IOException;
import java.lang.reflect.Constructor;
@@ -72,8 +73,8 @@ public class FragmentConvertor {
CONSTRUCTOR_CACHE.put(clazz, constructor);
}
result = constructor.newInstance(new Object[]{fragment.getContents()});
- } catch (Exception e) {
- throw new RuntimeException(e);
+ } catch (Throwable e) {
+ throw new TajoInternalError(e);
}
return result;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json
index 40e17f4..3ede2d4 100644
--- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json
+++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.json
@@ -12,9 +12,13 @@
"handler": "org.apache.tajo.storage.FileTablespace",
"default-format": "text"
},
- "hbase": {
+ "hbase:zk": {
"handler": "org.apache.tajo.storage.hbase.HBaseTablespace",
"default-format": "hbase"
+ },
+ "jdbc:postgresql": {
+ "handler": "org.apache.tajo.storage.pgsql.PgSQLTablespace",
+ "default-format": "rowstore"
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml
index 6b10b0b..7f4661b 100644
--- a/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml
+++ b/tajo-storage/tajo-storage-common/src/main/resources/storage-default.xml
@@ -87,6 +87,10 @@
<name>tajo.storage.fragment.hbase.class</name>
<value>org.apache.tajo.storage.hbase.HBaseFragment</value>
</property>
+ <property>
+ <name>tajo.storage.fragment.jdbc.class</name>
+ <value>org.apache.tajo.storage.jdbc.JdbcFragment</value>
+ </property>
<!--- Scanner Handler -->
<property>
@@ -143,7 +147,7 @@
<name>tajo.storage.scanner-handler.hbase.class</name>
<value>org.apache.tajo.storage.hbase.HBaseScanner</value>
</property>
-
+
<!--- Appender Handler -->
<property>
<name>tajo.storage.appender-handler</name>
@@ -231,4 +235,4 @@
<value>131072</value>
<description>128KB write buffer</description>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml
index 139133b..934dd01 100644
--- a/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml
+++ b/tajo-storage/tajo-storage-common/src/test/resources/storage-default.xml
@@ -86,6 +86,10 @@
<name>tajo.storage.fragment.hbase.class</name>
<value>org.apache.tajo.storage.hbase.HBaseFragment</value>
</property>
+ <property>
+ <name>tajo.storage.fragment.jdbc.class</name>
+ <value>org.apache.tajo.storage.jdbc.JdbcFragment</value>
+ </property>
<!--- Scanner Handler -->
<property>
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
index 7fc6d2a..90f7aa0 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
@@ -38,6 +38,7 @@ import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.datum.TextDatum;
import org.apache.tajo.exception.*;
import org.apache.tajo.plan.expr.EvalNode;
+import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.BytesUtils;
@@ -415,6 +416,11 @@ public class HBaseScanner implements Scanner {
}
@Override
+ public void pushOperators(LogicalNode planPart) {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ @Override
public boolean isProjectable() {
return true;
}
@@ -438,6 +444,10 @@ public class HBaseScanner implements Scanner {
}
@Override
+ public void setLimit(long num) {
+ }
+
+ @Override
public boolean isSplittable() {
return true;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
index 2204923..0064be4 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
@@ -18,7 +18,9 @@
package org.apache.tajo.storage.hbase;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
+import net.minidev.json.JSONObject;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -42,10 +44,7 @@ import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.TextDatum;
-import org.apache.tajo.exception.DataTypeMismatchException;
-import org.apache.tajo.exception.InvalidTablePropertyException;
-import org.apache.tajo.exception.MissingTablePropertyException;
-import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.exception.*;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.logical.CreateTableNode;
@@ -58,6 +57,7 @@ import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.*;
+import javax.annotation.Nullable;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -70,7 +70,8 @@ import java.util.*;
public class HBaseTablespace extends Tablespace {
private static final Log LOG = LogFactory.getLog(HBaseTablespace.class);
- public static final StorageProperty HBASE_STORAGE_PROPERTIES = new StorageProperty("hbase", false, true, false);
+ public static final StorageProperty HBASE_STORAGE_PROPERTIES =
+ new StorageProperty("hbase", false, true, false, false);
public static final FormatProperty HFILE_FORMAT_PROPERTIES = new FormatProperty(true, false, true);
public static final FormatProperty PUT_MODE_PROPERTIES = new FormatProperty(true, true, false);
@@ -80,8 +81,8 @@ public class HBaseTablespace extends Tablespace {
private Map<HConnectionKey, HConnection> connMap = new HashMap<HConnectionKey, HConnection>();
- public HBaseTablespace(String spaceName, URI uri) {
- super(spaceName, uri);
+ public HBaseTablespace(String spaceName, URI uri, JSONObject config) {
+ super(spaceName, uri, config);
}
@Override
@@ -93,21 +94,13 @@ public class HBaseTablespace extends Tablespace {
hbaseConf.set(HConstants.ZOOKEEPER_CLIENT_PORT, splits[1]);
}
- @Override
- public void setConfig(String name, String value) {
- }
-
- @Override
- public void setConfigs(Map<String, String> configs) {
- }
-
public Configuration getHbaseConf() {
return hbaseConf;
}
@Override
- public long getTableVolume(URI uri) throws IOException {
- return 0;
+ public long getTableVolume(URI uri) throws UnsupportedException {
+ throw new UnsupportedException();
}
@Override
@@ -424,12 +417,14 @@ public class HBaseTablespace extends Tablespace {
}
@Override
- public List<Fragment> getSplits(String fragmentId, TableDesc tableDesc, ScanNode scanNode)
+ public List<Fragment> getSplits(String inputSourceId,
+ TableDesc tableDesc,
+ @Nullable EvalNode filterCondition)
throws IOException, TajoException {
ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getOptions());
- List<IndexPredication> indexPredications = getIndexPredications(columnMapping, tableDesc, scanNode);
+ List<IndexPredication> indexPredications = getIndexPredications(columnMapping, tableDesc, filterCondition);
HTable htable = null;
HBaseAdmin hAdmin = null;
@@ -445,7 +440,7 @@ public class HBaseTablespace extends Tablespace {
List<Fragment> fragments = new ArrayList<Fragment>(1);
Fragment fragment = new HBaseFragment(
tableDesc.getUri(),
- fragmentId, htable.getName().getNameAsString(),
+ inputSourceId, htable.getName().getNameAsString(),
HConstants.EMPTY_BYTE_ARRAY,
HConstants.EMPTY_BYTE_ARRAY,
regLoc.getHostname());
@@ -523,7 +518,7 @@ public class HBaseTablespace extends Tablespace {
}
} else {
HBaseFragment fragment = new HBaseFragment(tableDesc.getUri(),
- fragmentId,
+ inputSourceId,
htable.getName().getNameAsString(),
fragmentStart,
fragmentStop,
@@ -713,14 +708,15 @@ public class HBaseTablespace extends Tablespace {
}
public List<IndexPredication> getIndexPredications(ColumnMapping columnMapping,
- TableDesc tableDesc, ScanNode scanNode)
+ TableDesc tableDesc,
+ @Nullable EvalNode filterCondition)
throws IOException, MissingTablePropertyException, InvalidTablePropertyException {
List<IndexPredication> indexPredications = new ArrayList<IndexPredication>();
Column[] indexableColumns = getIndexableColumns(tableDesc);
if (indexableColumns != null && indexableColumns.length == 1) {
// Currently supports only single index column.
- List<Set<EvalNode>> indexablePredicateList = findIndexablePredicateSet(scanNode, indexableColumns);
+ List<Set<EvalNode>> indexablePredicateList = findIndexablePredicateSet(filterCondition, indexableColumns);
for (Set<EvalNode> eachEvalSet: indexablePredicateList) {
Pair<Datum, Datum> indexPredicationValues = getIndexablePredicateValue(columnMapping, eachEvalSet);
if (indexPredicationValues != null) {
@@ -737,12 +733,13 @@ public class HBaseTablespace extends Tablespace {
return indexPredications;
}
- public List<Set<EvalNode>> findIndexablePredicateSet(ScanNode scanNode, Column[] indexableColumns) throws IOException {
+ public List<Set<EvalNode>> findIndexablePredicateSet(@Nullable EvalNode qual,
+ Column[] indexableColumns) throws IOException {
List<Set<EvalNode>> indexablePredicateList = new ArrayList<Set<EvalNode>>();
// if a query statement has a search condition, try to find indexable predicates
- if (indexableColumns != null && scanNode.getQual() != null) {
- EvalNode[] disjunctiveForms = AlgebraicUtil.toDisjunctiveNormalFormArray(scanNode.getQual());
+ if (indexableColumns != null && qual != null) {
+ EvalNode[] disjunctiveForms = AlgebraicUtil.toDisjunctiveNormalFormArray(qual);
// add qualifier to schema for qual
for (Column column : indexableColumns) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java
index f0c8f15..56ca9be 100644
--- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java
+++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestHBaseTableSpace.java
@@ -43,7 +43,7 @@ public class TestHBaseTableSpace {
@BeforeClass
public static void setUp() throws IOException {
String tableSpaceUri = "hbase:zk://host1:2171";
- HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri));
+ HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri), null);
hBaseTablespace.init(new TajoConf());
TablespaceManager.addTableSpaceForTest(hBaseTablespace);
}
@@ -74,7 +74,8 @@ public class TestHBaseTableSpace {
scanNode.setQual(evalNodeA);
HBaseTablespace storageManager = (HBaseTablespace) TablespaceManager.getByName("cluster1").get();
- List<Set<EvalNode>> indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
+ List<Set<EvalNode>> indexEvals =
+ storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn});
assertNotNull(indexEvals);
assertEquals(1, indexEvals.size());
Pair<Datum, Datum> indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
@@ -85,7 +86,7 @@ public class TestHBaseTableSpace {
EvalNode evalNode3 = new BinaryEval(EvalType.EQUAL, new FieldEval(rowkeyColumn),new ConstEval(new TextDatum("075")));
EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3);
scanNode.setQual(evalNodeB);
- indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
+ indexEvals = storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn});
assertEquals(2, indexEvals.size());
indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
assertEquals("020", indexPredicateValue.getFirst().asChars());
@@ -101,7 +102,7 @@ public class TestHBaseTableSpace {
EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5);
EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC);
scanNode.setQual(evalNodeD);
- indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
+ indexEvals = storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn});
assertEquals(2, indexEvals.size());
indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
@@ -120,7 +121,7 @@ public class TestHBaseTableSpace {
evalNodeD = new BinaryEval(EvalType.AND, evalNodeC, evalNode6);
EvalNode evalNodeE = new BinaryEval(EvalType.OR, evalNodeA, evalNodeD);
scanNode.setQual(evalNodeE);
- indexEvals = storageManager.findIndexablePredicateSet(scanNode, new Column[]{rowkeyColumn});
+ indexEvals = storageManager.findIndexablePredicateSet(scanNode.getQual(), new Column[]{rowkeyColumn});
assertEquals(2, indexEvals.size());
indexPredicateValue = storageManager.getIndexablePredicateValue(null, indexEvals.get(0));
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
index 8844fa5..33f1d04 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
@@ -29,6 +29,9 @@ import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.exception.TajoRuntimeException;
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
@@ -82,6 +85,11 @@ public abstract class FileScanner implements Scanner {
}
@Override
+ public void pushOperators(LogicalNode planPart) {
+ throw new TajoRuntimeException(new UnsupportedException());
+ }
+
+ @Override
public void setTarget(Column[] targets) {
if (inited) {
throw new IllegalStateException("Should be called before init()");
@@ -89,6 +97,10 @@ public abstract class FileScanner implements Scanner {
this.targets = targets;
}
+ @Override
+ public void setLimit(long num) {
+ }
+
public static FileSystem getFileSystem(TajoConf tajoConf, Path path) throws IOException {
String tajoUser = tajoConf.getVar(TajoConf.ConfVars.USERNAME);
FileSystem fs;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
index f4c78b6..af981bb 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import net.minidev.json.JSONObject;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -34,15 +35,18 @@ import org.apache.tajo.*;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.NodeType;
-import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.TUtil;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.net.URI;
import java.text.NumberFormat;
@@ -94,7 +98,7 @@ public class FileTablespace extends Tablespace {
}
};
- private static final StorageProperty FileStorageProperties = new StorageProperty("TEXT", true, true, true);
+ private static final StorageProperty FileStorageProperties = new StorageProperty("TEXT", true, true, true, false);
private static final FormatProperty GeneralFileProperties = new FormatProperty(true, false, true);
protected FileSystem fs;
@@ -103,8 +107,8 @@ public class FileTablespace extends Tablespace {
protected boolean blocksMetadataEnabled;
private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0));
- public FileTablespace(String spaceName, URI uri) {
- super(spaceName, uri);
+ public FileTablespace(String spaceName, URI uri, JSONObject config) {
+ super(spaceName, uri, config);
}
@Override
@@ -123,21 +127,14 @@ public class FileTablespace extends Tablespace {
}
@Override
- public void setConfig(String name, String value) {
- conf.set(name, value);
- }
-
- @Override
- public void setConfigs(Map<String, String> configs) {
- for (Map.Entry<String, String> c : configs.entrySet()) {
- conf.set(c.getKey(), c.getValue());
- }
- }
-
- @Override
- public long getTableVolume(URI uri) throws IOException {
+ public long getTableVolume(URI uri) throws UnsupportedException {
Path path = new Path(uri);
- ContentSummary summary = fs.getContentSummary(path);
+ ContentSummary summary;
+ try {
+ summary = fs.getContentSummary(path);
+ } catch (IOException e) {
+ throw new TajoInternalError(e);
+ }
return summary.getLength();
}
@@ -155,7 +152,7 @@ public class FileTablespace extends Tablespace {
public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status)
throws IOException {
Fragment fragment = new FileFragment(path.getName(), path, 0, status.getLen());
- return getScanner(meta, schema, fragment);
+ return getScanner(meta, schema, fragment, null);
}
public FileSystem getFileSystem() {
@@ -627,8 +624,10 @@ public class FileTablespace extends Tablespace {
}
@Override
- public List<Fragment> getSplits(String tableName, TableDesc table, ScanNode scanNode) throws IOException {
- return getSplits(tableName, table.getMeta(), table.getSchema(), new Path(table.getUri()));
+ public List<Fragment> getSplits(String inputSourceId,
+ TableDesc table,
+ @Nullable EvalNode filterCondition) throws IOException {
+ return getSplits(inputSourceId, table.getMeta(), table.getSchema(), new Path(table.getUri()));
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
index eabab22..76c9362 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
@@ -25,6 +25,7 @@ import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.storage.FieldSerializerDeserializer;
import org.apache.tajo.storage.Tuple;
@@ -41,12 +42,7 @@ public class CSVLineDeserializer extends TextLineDeserializer {
public CSVLineDeserializer(Schema schema, TableMeta meta, Column [] projected) {
super(schema, meta);
-
- targetColumnIndexes = new int[projected.length];
- for (int i = 0; i < projected.length; i++) {
- targetColumnIndexes[i] = schema.getColumnId(projected[i].getQualifiedName());
- }
- Arrays.sort(targetColumnIndexes);
+ targetColumnIndexes = PlannerUtil.getTargetIds(schema, projected);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
index 90bec65..5f8a4d1 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestDelimitedTextFile.java
@@ -106,7 +106,7 @@ public class TestDelimitedTextFile {
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "-1");
FileFragment fragment = getFileFragment("testErrorTolerance1.json");
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple tuple;
@@ -128,7 +128,7 @@ public class TestDelimitedTextFile {
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1");
FileFragment fragment = getFileFragment("testErrorTolerance1.json");
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
assertNotNull(scanner.next());
@@ -150,7 +150,7 @@ public class TestDelimitedTextFile {
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "0");
FileFragment fragment = getFileFragment("testErrorTolerance2.json");
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
try {
@@ -169,7 +169,7 @@ public class TestDelimitedTextFile {
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1");
FileFragment fragment = getFileFragment("testErrorTolerance3.json");
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
try {
@@ -185,7 +185,7 @@ public class TestDelimitedTextFile {
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageConstants.TEXT_SKIP_HEADER_LINE, "2");
FileFragment fragment = getFileFragment("testNormal.json");
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
@@ -212,7 +212,7 @@ public class TestDelimitedTextFile {
meta.putOption(StorageConstants.TEXT_SKIP_HEADER_LINE, "1");
meta.putOption(StorageConstants.TEXT_DELIMITER, ",");
FileFragment fragment = getFileFragment("testSkip.txt");
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
index cecd4dd..f536514 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileTablespace.java
@@ -20,6 +20,7 @@ package org.apache.tajo.storage;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
+import net.minidev.json.JSONObject;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -128,7 +129,7 @@ public class TestFileTablespace {
}
assertTrue(fs.exists(tablePath));
- FileTablespace space = new FileTablespace("testGetSplit", fs.getUri());
+ FileTablespace space = new FileTablespace("testGetSplit", fs.getUri(), null);
space.init(new TajoConf(conf));
assertEquals(fs.getUri(), space.getUri());
@@ -186,7 +187,7 @@ public class TestFileTablespace {
}
assertTrue(fs.exists(tablePath));
- FileTablespace space = new FileTablespace("testZeroLengthSplit", fs.getUri());
+ FileTablespace space = new FileTablespace("testZeroLengthSplit", fs.getUri(), new JSONObject());
space.init(new TajoConf(conf));
assertEquals(fs.getUri(), space.getUri());
@@ -233,7 +234,7 @@ public class TestFileTablespace {
}
assertTrue(fs.exists(tablePath));
- FileTablespace sm = new FileTablespace("testGetSplitWithBlockStorageLocationsBatching", fs.getUri());
+ FileTablespace sm = new FileTablespace("testGetSplitWithBlockStorageLocationsBatching", fs.getUri(), null);
sm.init(new TajoConf(conf));
assertEquals(fs.getUri(), sm.getUri());
@@ -276,7 +277,7 @@ public class TestFileTablespace {
FileTablespace space = TablespaceManager.getLocalFs();
assertEquals(localFs.getUri(), space.getFileSystem().getUri());
- FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri);
+ FileTablespace distTablespace = new FileTablespace("testGetFileTablespace", uri, null);
distTablespace.init(conf);
existingTs = TablespaceManager.addTableSpaceForTest(distTablespace);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
index 21fff58..b800ed2 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestLineReader.java
@@ -38,6 +38,7 @@ import org.apache.tajo.storage.text.DelimitedLineReader;
import org.apache.tajo.storage.text.DelimitedTextFile;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.junit.Test;
import java.io.File;
@@ -213,7 +214,7 @@ public class TestLineReader {
@Test
public void testByteBufLineReaderWithoutTerminating() throws IOException {
- String path = FileUtil.getResourcePath("dataset/testLineText.txt").getFile();
+ String path = JavaResourceUtil.getResourceURL("dataset/testLineText.txt").getFile();
File file = new File(path);
String data = FileUtil.readTextFile(file);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index 02472eb..26e16ea 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -44,7 +44,7 @@ import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.rcfile.RCFile;
import org.apache.tajo.storage.sequencefile.SequenceFileScanner;
import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.KeyValueSet;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -64,36 +64,36 @@ public class TestStorages {
private static String TEST_PROJECTION_AVRO_SCHEMA =
"{\n" +
- " \"type\": \"record\",\n" +
- " \"namespace\": \"org.apache.tajo\",\n" +
- " \"name\": \"testProjection\",\n" +
- " \"fields\": [\n" +
- " { \"name\": \"id\", \"type\": \"int\" },\n" +
- " { \"name\": \"age\", \"type\": \"long\" },\n" +
- " { \"name\": \"score\", \"type\": \"float\" }\n" +
- " ]\n" +
- "}\n";
+ " \"type\": \"record\",\n" +
+ " \"namespace\": \"org.apache.tajo\",\n" +
+ " \"name\": \"testProjection\",\n" +
+ " \"fields\": [\n" +
+ " { \"name\": \"id\", \"type\": \"int\" },\n" +
+ " { \"name\": \"age\", \"type\": \"long\" },\n" +
+ " { \"name\": \"score\", \"type\": \"float\" }\n" +
+ " ]\n" +
+ "}\n";
private static String TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA =
"{\n" +
- " \"type\": \"record\",\n" +
- " \"namespace\": \"org.apache.tajo\",\n" +
- " \"name\": \"testNullHandlingTypes\",\n" +
- " \"fields\": [\n" +
- " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" +
- " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" +
- " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" +
- " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" +
- " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" +
- " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" +
- " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col11\", \"type\": \"null\" },\n" +
- " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" +
- " ]\n" +
- "}\n";
+ " \"type\": \"record\",\n" +
+ " \"namespace\": \"org.apache.tajo\",\n" +
+ " \"name\": \"testNullHandlingTypes\",\n" +
+ " \"fields\": [\n" +
+ " { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" +
+ " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" +
+ " { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" +
+ " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" +
+ " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" +
+ " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" +
+ " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" +
+ " { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" +
+ " { \"name\": \"col11\", \"type\": \"null\" },\n" +
+ " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" +
+ " ]\n" +
+ "}\n";
private static String TEST_MAX_VALUE_AVRO_SCHEMA =
"{\n" +
@@ -392,7 +392,7 @@ public class TestStorages {
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.setOptions(CatalogUtil.newDefaultProperty(storeType));
if (storeType.equalsIgnoreCase(BuiltinStorages.AVRO)) {
- String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString();
+ String path = JavaResourceUtil.getResourceURL("dataset/testVariousTypes.avsc").toString();
meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
}
@@ -429,7 +429,7 @@ public class TestStorages {
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = sm.getScanner(meta, schema, fragment);
+ Scanner scanner = sm.getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
@@ -518,7 +518,7 @@ public class TestStorages {
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
@@ -593,7 +593,7 @@ public class TestStorages {
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
@@ -662,7 +662,7 @@ public class TestStorages {
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
@@ -731,7 +731,7 @@ public class TestStorages {
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
assertTrue(scanner instanceof SequenceFileScanner);
@@ -805,7 +805,7 @@ public class TestStorages {
assertEquals(appender.getStats().getNumBytes().longValue(), status.getLen());
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
assertTrue(scanner instanceof SequenceFileScanner);
@@ -850,7 +850,7 @@ public class TestStorages {
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
@@ -986,7 +986,7 @@ public class TestStorages {
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = sm.getScanner(meta, schema, fragment);
+ Scanner scanner = sm.getScanner(meta, schema, fragment, null);
scanner.init();
Tuple retrieved;
@@ -1048,7 +1048,7 @@ public class TestStorages {
inSchema.addColumn("col5", Type.INT8);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, inSchema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, inSchema, fragment, null);
Schema target = new Schema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
index 341cc07..960448c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
@@ -25,6 +25,7 @@ import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.NetUtils;
import org.junit.Before;
import org.junit.Test;
@@ -46,7 +47,7 @@ public class TestAvroUtil {
@Before
public void setUp() throws Exception {
- schemaUrl = FileUtil.getResourcePath("dataset/testVariousTypes.avsc");
+ schemaUrl = JavaResourceUtil.getResourceURL("dataset/testVariousTypes.avsc");
assertNotNull(schemaUrl);
File file = new File(schemaUrl.getPath());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
index 88d7536..75e59da 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
@@ -69,7 +69,7 @@ public class TestJsonSerDe {
FileSystem fs = FileSystem.getLocal(conf);
FileStatus status = fs.getFileStatus(tablePath);
FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple tuple = scanner.next();
@@ -108,7 +108,7 @@ public class TestJsonSerDe {
schema.addColumn("col1", TajoDataTypes.Type.TEXT);
schema.addColumn("col2", TajoDataTypes.Type.TEXT);
schema.addColumn("col3", TajoDataTypes.Type.TEXT);
- Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
+ Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null);
scanner.init();
Tuple tuple = scanner.next();
[4/4] tajo git commit: TAJO-1730: JDBC Tablespace support.
Posted by hy...@apache.org.
TAJO-1730: JDBC Tablespace support.
Closes #719
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0e4ad563
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0e4ad563
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0e4ad563
Branch: refs/heads/master
Commit: 0e4ad5635be05bdde8dac5feb6c9a85fb3cf0e06
Parents: e1c2d35
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Sep 17 05:40:58 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Sep 17 05:40:58 2015 -0700
----------------------------------------------------------------------
CHANGES | 2 +
.../apache/tajo/catalog/MetadataProvider.java | 40 ++-
.../java/org/apache/tajo/catalog/TableUtil.java | 51 ---
.../apache/tajo/catalog/TestFunctionDesc.java | 19 --
.../org/apache/tajo/catalog/CatalogServer.java | 15 +-
.../tajo/catalog/LinkedMetadataManager.java | 23 +-
.../tajo/catalog/store/AbstractDBStore.java | 9 +-
.../tajo/catalog/TestLinkedMetadataManager.java | 12 +-
.../tajo/client/CatalogAdminClientImpl.java | 8 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 17 +-
.../org/apache/tajo/TajoTestingCluster.java | 2 +-
.../test/java/org/apache/tajo/TpchTestBase.java | 4 +-
.../apache/tajo/exception/ExceptionUtil.java | 25 +-
.../tajo/exception/NotImplementedException.java | 3 +-
.../tajo/exception/UnsupportedException.java | 2 +-
.../org/apache/tajo/storage/StorageService.java | 4 +
.../java/org/apache/tajo/util/FileUtil.java | 116 ++-----
.../org/apache/tajo/util/JavaResourceUtil.java | 50 +++
.../java/org/apache/tajo/util/StringUtils.java | 37 +-
.../main/java/org/apache/tajo/util/UriUtil.java | 46 +++
.../org/apache/tajo/util/TestFileUtils.java | 107 ------
.../java/org/apache/tajo/util/TestUriUtil.java | 43 +++
.../apache/tajo/cli/tools/TestDDLBuilder.java | 10 +-
.../tajo/engine/query/TestHBaseTable.java | 12 +-
.../apache/tajo/engine/query/TestJoinQuery.java | 4 +-
.../apache/tajo/parser/sql/TestSQLAnalyzer.java | 6 +-
.../org/apache/tajo/storage/TestRowFile.java | 8 +-
.../tajo/engine/codegen/EvalCodeGenerator.java | 4 +-
.../engine/planner/physical/SeqScanExec.java | 12 +-
.../java/org/apache/tajo/master/TajoMaster.java | 18 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../apache/tajo/master/exec/QueryExecutor.java | 1 +
.../apache/tajo/querymaster/Repartitioner.java | 23 +-
.../java/org/apache/tajo/querymaster/Stage.java | 2 +-
tajo-dist/pom.xml | 3 +-
tajo-docs/src/main/sphinx/index.rst | 2 +-
tajo-docs/src/main/sphinx/storage_plugin.rst | 47 ---
tajo-docs/src/main/sphinx/storage_plugins.rst | 11 +
.../main/sphinx/storage_plugins/overview.rst | 47 +++
.../main/sphinx/storage_plugins/postgresql.rst | 40 +++
.../org/apache/tajo/plan/LogicalPlanner.java | 28 +-
.../apache/tajo/plan/expr/AlgebraicUtil.java | 2 +-
.../org/apache/tajo/plan/expr/EvalType.java | 14 +-
.../tajo/plan/expr/SimpleEvalNodeVisitor.java | 13 +-
.../plan/exprrewrite/rules/ConstantFolding.java | 2 +-
.../exprrewrite/rules/ConstantPropagation.java | 2 +-
.../org/apache/tajo/plan/logical/ScanNode.java | 30 +-
.../rules/CommonConditionReduceRule.java | 2 +-
.../tajo/plan/serder/EvalNodeSerializer.java | 6 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 44 ++-
tajo-plan/src/main/proto/Plan.proto | 8 +-
tajo-project/pom.xml | 11 +
tajo-storage/pom.xml | 2 +
.../apache/tajo/storage/AbstractScanner.java | 10 +
.../org/apache/tajo/storage/MergeScanner.java | 10 +
.../org/apache/tajo/storage/NullScanner.java | 11 +
.../apache/tajo/storage/OldStorageManager.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 16 +
.../apache/tajo/storage/StorageProperty.java | 15 +-
.../org/apache/tajo/storage/Tablespace.java | 98 +++---
.../apache/tajo/storage/TablespaceManager.java | 84 +++--
.../storage/fragment/FragmentConvertor.java | 5 +-
.../src/main/resources/storage-default.json | 6 +-
.../src/main/resources/storage-default.xml | 8 +-
.../src/test/resources/storage-default.xml | 4 +
.../apache/tajo/storage/hbase/HBaseScanner.java | 10 +
.../tajo/storage/hbase/HBaseTablespace.java | 49 ++-
.../tajo/storage/hbase/TestHBaseTableSpace.java | 11 +-
.../org/apache/tajo/storage/FileScanner.java | 12 +
.../org/apache/tajo/storage/FileTablespace.java | 41 ++-
.../tajo/storage/text/CSVLineDeserializer.java | 8 +-
.../tajo/storage/TestDelimitedTextFile.java | 12 +-
.../apache/tajo/storage/TestFileTablespace.java | 9 +-
.../org/apache/tajo/storage/TestLineReader.java | 3 +-
.../org/apache/tajo/storage/TestStorages.java | 76 ++---
.../apache/tajo/storage/avro/TestAvroUtil.java | 3 +-
.../apache/tajo/storage/json/TestJsonSerDe.java | 4 +-
tajo-storage/tajo-storage-jdbc/pom.xml | 243 ++++++++++++++
.../tajo/storage/jdbc/ConnectionInfo.java | 115 +++++++
.../apache/tajo/storage/jdbc/JdbcFragment.java | 106 ++++++
.../storage/jdbc/JdbcMetadataProviderBase.java | 252 ++++++++++++++
.../apache/tajo/storage/jdbc/JdbcScanner.java | 334 +++++++++++++++++++
.../tajo/storage/jdbc/JdbcTablespace.java | 209 ++++++++++++
.../apache/tajo/storage/jdbc/SQLBuilder.java | 188 +++++++++++
.../storage/jdbc/SQLExpressionGenerator.java | 302 +++++++++++++++++
.../src/main/proto/JdbcFragmentProtos.proto | 31 ++
.../tajo/storage/jdbc/TestConnectionInfo.java | 52 +++
tajo-storage/tajo-storage-pgsql/pom.xml | 257 ++++++++++++++
.../tajo/storage/pgsql/PgSQLJdbcScanner.java | 38 +++
.../storage/pgsql/PgSQLMetadataProvider.java | 41 +++
.../tajo/storage/pgsql/PgSQLTablespace.java | 71 ++++
.../tajo/storage/pgsql/PgSQLTestServer.java | 211 ++++++++++++
.../storage/pgsql/TestPgSQLEndPointTests.java | 74 ++++
.../storage/pgsql/TestPgSQLJdbcTableSpace.java | 127 +++++++
.../pgsql/TestPgSQLMetadataProvider.java | 84 +++++
.../tajo/storage/pgsql/TestPgSQLQueryTests.java | 200 +++++++++++
.../pgsql/TestPgSQLSimpleQueryTests.java | 48 +++
.../src/test/resources/dataset/.marker | 1 +
.../src/test/resources/pgsql/customer.sql | 10 +
.../src/test/resources/pgsql/datetime_types.sql | 5 +
.../src/test/resources/pgsql/datetime_types.txt | 4 +
.../src/test/resources/pgsql/lineitem.sql | 18 +
.../src/test/resources/pgsql/nation.sql | 6 +
.../src/test/resources/pgsql/orders.sql | 11 +
.../src/test/resources/pgsql/part.sql | 11 +
.../src/test/resources/pgsql/partsupp.sql | 7 +
.../src/test/resources/pgsql/region.sql | 5 +
.../src/test/resources/pgsql/supplier.sql | 9 +
.../TestPgSQLQueryTests/testBetweenDates.sql | 1 +
.../TestPgSQLQueryTests/testBetweenNumbers.sql | 1 +
.../TestPgSQLQueryTests/testCaseWhenFilter.sql | 6 +
.../TestPgSQLQueryTests/testCountAsterisk.sql | 1 +
.../TestPgSQLQueryTests/testCtasToHdfs.sql | 1 +
.../TestPgSQLQueryTests/testDateTimeTypes.sql | 1 +
.../testFixedLengthFields.sql | 1 +
.../testFunctionWithinFilter.sql | 1 +
.../testInPredicateWithLiterals.sql | 1 +
.../testInPredicateWithNumbers.sql | 1 +
.../testProjectedColumns.sql | 1 +
.../testQueryWithConnProperties.sql | 1 +
.../TestPgSQLQueryTests/testSimpleFilter.sql | 1 +
.../queries/TestPgSQLQueryTests/testTPCH_Q1.sql | 21 ++
.../TestPgSQLQueryTests/testTPCH_Q2_Part.sql | 18 +
.../testTPCH_Q2_Part_MixedStorage.sql | 18 +
.../queries/TestPgSQLQueryTests/testTPCH_Q3.sql | 22 ++
.../queries/TestPgSQLQueryTests/testTPCH_Q5.sql | 25 ++
.../testVariableLengthFields.sql | 1 +
.../TestPgSQLSimpleQueryTests/testSelectAll.sql | 1 +
.../testSelectLimit.sql | 1 +
.../TestPgSQLQueryTests/testBetweenDates.result | 5 +
.../testBetweenNumbers.result | 5 +
.../testCaseWhenFilter.result | 5 +
.../testCountAsterisk.result | 3 +
.../TestPgSQLQueryTests/testCtasToHdfs.result | 7 +
.../testDateTimeTypes.result | 6 +
.../testFixedLengthFields.result | 5 +
.../testFunctionWithinFilter.result | 5 +
.../testInPredicateWithLiterals.result | 5 +
.../testInPredicateWithNumbers.result | 5 +
.../testProjectedColumns.result | 5 +
.../testQueryWithConnProperties.result | 5 +
.../TestPgSQLQueryTests/testSimpleFilter.result | 5 +
.../TestPgSQLQueryTests/testTPCH_Q1.result | 4 +
.../TestPgSQLQueryTests/testTPCH_Q2_Part.result | 3 +
.../testTPCH_Q2_Part_MixedStorage.result | 3 +
.../TestPgSQLQueryTests/testTPCH_Q3.result | 5 +
.../TestPgSQLQueryTests/testTPCH_Q5.result | 27 ++
.../testVariableLengthFields.result | 5 +
.../testSelectAll.result | 7 +
.../testSelectLimit.result | 5 +
150 files changed, 4200 insertions(+), 678 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 122f895..0c201f1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,8 @@ Release 0.11.0 - unreleased
NEW FEATURES
+ TAJO-1730: JDBC Tablespace support. (hyunsik)
+
TAJO-1812: Timezone support in JSON file format. (hyunsik)
TAJO-1486: Text file should support to skip header rows when creating
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
index cafe0a1..f0d0e39 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
@@ -25,15 +25,49 @@ import java.net.URI;
import java.util.Collection;
public interface MetadataProvider {
+ /**
+ * Get a Tablespace name
+ *
+ * @return Tablespace name
+ */
String getTablespaceName();
+ /**
+ * Get a tablespace URI
+ *
+ * @return Table space URI
+ */
URI getTablespaceUri();
+ /**
+ * Return a database name
+ *
+ * @return Database name
+ */
String getDatabaseName();
- Collection<String> getCatalogs();
+ /**
+ * Get a list of schema names
+ * @return All schema names in this database
+ */
+ Collection<String> getSchemas();
- Collection<String> getTables(@Nullable String catalog);
+ /**
+ * get Table names matched to a given patterns
+ *
+ * @param schemaPattern Schema name pattern
+ * @param tablePattern Table name pattern
+ * @return All matched table names
+ */
+ Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern);
- TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException;
+ /**
+ * Get a table descriptor
+ *
+ * @param schemaName schema name
+ * @param tableName table name
+ * @return TableDesc
+ * @throws UndefinedTablespaceException
+ */
+ TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
deleted file mode 100644
index 76aec33..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.util.FileUtil;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-public class TableUtil {
- public static TableMeta getTableMeta(Configuration conf, Path tablePath)
- throws IOException {
- TableMeta meta = null;
-
- FileSystem fs = tablePath.getFileSystem(conf);
-
- Path tableMetaPath = new Path(tablePath, ".meta");
- if(!fs.exists(tableMetaPath)) {
- throw new FileNotFoundException(".meta file not found in "+tablePath.toString());
- }
- FSDataInputStream tableMetaIn =
- fs.open(tableMetaPath);
-
- TableProto tableProto = (TableProto) FileUtil.loadProto(tableMetaIn,
- TableProto.getDefaultInstance());
- meta = new TableMeta(tableProto);
-
- return meta;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
index 16e85f0..e8170ee 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
@@ -71,25 +71,6 @@ public class TestFunctionDesc {
assertEquals(Type.INT4, desc.getReturnType().getType());
assertArrayEquals(CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8),
desc.getParamTypes());
-
- CommonTestingUtil.getTestDir(TEST_PATH);
- File save = new File(TEST_PATH + "/save.dat");
- FileUtil.writeProto(save, desc.getProto());
-
- FunctionDescProto proto = FunctionDescProto.getDefaultInstance();
- proto = (FunctionDescProto) FileUtil.loadProto(save, proto);
-
- FunctionDesc newDesc = new FunctionDesc(proto);
-
- assertEquals("sum", newDesc.getFunctionName());
- assertEquals(TestSum.class, newDesc.getLegacyFuncClass());
- assertEquals(FunctionType.GENERAL, newDesc.getFuncType());
- assertEquals(Type.INT4, newDesc.getReturnType().getType());
-
- assertArrayEquals(CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8),
- newDesc.getParamTypes());
-
- assertEquals(desc.getProto(), newDesc.getProto());
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index dff292f..8f4e0e8 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -18,6 +18,7 @@
package org.apache.tajo.catalog;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
@@ -84,7 +85,7 @@ public class CatalogServer extends AbstractService {
private Map<String, List<FunctionDescProto>> functions = new ConcurrentHashMap<String,
List<FunctionDescProto>>();
- protected final LinkedMetadataManager linkedMetadataManager;
+ protected LinkedMetadataManager linkedMetadataManager;
protected final InfoSchemaMetadataDictionary metaDictionary = new InfoSchemaMetadataDictionary();
// RPC variables
@@ -102,7 +103,8 @@ public class CatalogServer extends AbstractService {
this.builtingFuncs = new ArrayList<FunctionDesc>();
}
- public CatalogServer(Set<MetadataProvider> metadataProviders, Collection<FunctionDesc> sqlFuncs) throws IOException {
+ public CatalogServer(Collection<MetadataProvider> metadataProviders, Collection<FunctionDesc> sqlFuncs)
+ throws IOException {
super(CatalogServer.class.getName());
this.handler = new CatalogProtocolHandler();
this.linkedMetadataManager = new LinkedMetadataManager(metadataProviders);
@@ -192,6 +194,15 @@ public class CatalogServer extends AbstractService {
super.serviceStop();
}
+ /**
+ * Refresh the linked metadata manager. This must be used for only testing.
+ * @param metadataProviders
+ */
+ @VisibleForTesting
+ public void refresh(Collection<MetadataProvider> metadataProviders) {
+ this.linkedMetadataManager = new LinkedMetadataManager(metadataProviders);
+ }
+
public CatalogProtocolHandler getHandler() {
return this.handler;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
index 6d4af1f..a4c718e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
@@ -32,7 +32,6 @@ import javax.annotation.Nullable;
import java.net.URI;
import java.util.Collection;
import java.util.Map;
-import java.util.regex.Pattern;
import static com.google.common.collect.Collections2.filter;
@@ -163,7 +162,7 @@ public class LinkedMetadataManager {
public Collection<String> getSchemas(@Nullable String dbName) throws UndefinedDatabaseException {
ensureIfDBExists(dbName);
- return providerMap.get(dbName).getCatalogs();
+ return providerMap.get(dbName).getSchemas();
}
/**
@@ -180,21 +179,9 @@ public class LinkedMetadataManager {
*/
public Collection<String> getTableNames(String dbName,
@Nullable final String schemaPattern,
- final String tablePattern) throws UndefinedDatabaseException {
+ @Nullable final String tablePattern) throws UndefinedDatabaseException {
ensureIfDBExists(dbName);
-
- if (tablePattern == null) { // all tables in this database
- return providerMap.get(dbName).getTables("null");
-
- } else {
- final Pattern pattern = Pattern.compile(tablePattern);
- return filter(providerMap.get(dbName).getTables(schemaPattern), new Predicate<String>() {
- @Override
- public boolean apply(@Nullable String input) {
- return pattern.matcher(tablePattern).matches();
- }
- });
- }
+ return providerMap.get(dbName).getTables(schemaPattern, tablePattern);
}
/**
@@ -232,7 +219,7 @@ public class LinkedMetadataManager {
*/
public boolean existsTable(String dbName, String schemaName, String tableName) {
if (providerMap.containsKey(dbName)) {
- return providerMap.get(dbName).getTables(schemaName).contains(tableName);
+ return providerMap.get(dbName).getTables(schemaName, tableName).contains(tableName);
}
return false;
@@ -251,6 +238,6 @@ public class LinkedMetadataManager {
ensureIfDBExists(dbName);
- return providerMap.get(dbName).getTableDescriptor(schemaName, tbName);
+ return providerMap.get(dbName).getTableDesc(schemaName, tbName);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index 0b1b120..0f07149 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -33,14 +33,17 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.*;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import java.io.IOException;
import java.net.URI;
import java.sql.*;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto;
@@ -198,7 +201,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
public String readSchemaFile(String path) {
try {
- return FileUtil.readTextFileFromResource("schemas/" + path);
+ return JavaResourceUtil.readTextFromResource("schemas/" + path);
} catch (IOException e) {
throw new TajoInternalError(e);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
index 30c563f..a065fa8 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
@@ -82,17 +82,17 @@ public class TestLinkedMetadataManager {
}
@Override
- public Collection<String> getCatalogs() {
+ public Collection<String> getSchemas() {
return Lists.newArrayList("cat1", "cat2");
}
@Override
- public Collection<String> getTables(@Nullable String catalog) {
+ public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) {
return Lists.newArrayList("table1", "table2");
}
@Override
- public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException {
+ public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException {
if (tableName.equals("table1")) {
return TABLE1;
} else if (tableName.equals("table2")) {
@@ -121,17 +121,17 @@ public class TestLinkedMetadataManager {
}
@Override
- public Collection<String> getCatalogs() {
+ public Collection<String> getSchemas() {
return Lists.newArrayList("cat3", "cat4");
}
@Override
- public Collection<String> getTables(@Nullable String catalog) {
+ public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) {
return Lists.newArrayList("table3", "table4");
}
@Override
- public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException {
+ public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException {
if (tableName.equals("table3")) {
return TABLE3;
} else if (tableName.equals("table4")) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
index fb13f07..102b39e 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
@@ -87,15 +87,15 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
}
@Override
- public void dropDatabase(final String databaseName) throws UndefinedDatabaseException {
+ public void dropDatabase(final String databaseName)
+ throws UndefinedDatabaseException, InsufficientPrivilegeException {
try {
final BlockingInterface stub = conn.getTMStub();
final ReturnState state = stub.dropDatabase(null, conn.getSessionedString(databaseName));
- if (isThisError(state, ResultCode.UNDEFINED_DATABASE)) {
- throw new UndefinedDatabaseException(state);
- }
+ throwsIfThisError(state, UndefinedDatabaseException.class);
+ throwsIfThisError(state, InsufficientPrivilegeException.class);
ensureOk(state);
} catch (ServiceException e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index 5a912e5..9abfbd3 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -19,6 +19,7 @@
package org.apache.tajo;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.protobuf.ServiceException;
@@ -39,6 +40,7 @@ import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.exception.InsufficientPrivilegeException;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.UndefinedTableException;
import org.apache.tajo.jdbc.FetchResultSet;
@@ -206,10 +208,13 @@ public class QueryTestCaseBase {
client = testBase.getTestingCluster().newTajoClient();
URL datasetBaseURL = ClassLoader.getSystemResource("dataset");
+ Preconditions.checkNotNull(datasetBaseURL, "dataset directory is absent.");
datasetBasePath = new Path(datasetBaseURL.toString());
URL queryBaseURL = ClassLoader.getSystemResource("queries");
+ Preconditions.checkNotNull(queryBaseURL, "queries directory is absent.");
queryBasePath = new Path(queryBaseURL.toString());
URL resultBaseURL = ClassLoader.getSystemResource("results");
+ Preconditions.checkNotNull(resultBaseURL, "results directory is absent.");
resultBasePath = new Path(resultBaseURL.toString());
}
@@ -223,11 +228,19 @@ public class QueryTestCaseBase {
// if the current database is "default", shouldn't drop it.
if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
for (String tableName : catalog.getAllTableNames(currentDatabase)) {
- client.updateQuery("DROP TABLE IF EXISTS " + tableName);
+ try {
+ client.updateQuery("DROP TABLE IF EXISTS " + tableName);
+ } catch (InsufficientPrivilegeException i) {
+ LOG.warn("relation '" + tableName + "' is read only.");
+ }
}
client.selectDatabase(TajoConstants.DEFAULT_DATABASE_NAME);
- client.dropDatabase(currentDatabase);
+ try {
+ client.dropDatabase(currentDatabase);
+ } catch (InsufficientPrivilegeException e) {
+ LOG.warn("database '" + currentDatabase + "' is read only.");
+ }
}
client.close();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 680b5ec..8d2ee8f 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -345,7 +345,7 @@ public class TajoTestingCluster {
URI defaultTsUri = TajoConf.getWarehouseDir(c).toUri();
FileTablespace defaultTableSpace =
- new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri);
+ new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri, null);
defaultTableSpace.init(conf);
TablespaceManager.addTableSpaceForTest(defaultTableSpace);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
index 027e735..62f7477 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
@@ -27,10 +27,10 @@ import org.apache.tajo.catalog.Schema;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.KeyValueSet;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.ResultSet;
import java.util.Map;
@@ -76,7 +76,7 @@ public class TpchTestBase {
File tpchTablesDir = new File(new File(CommonTestingUtil.getTestDir().toUri()), "tpch");
for (int i = 0; i < names.length; i++) {
- String str = FileUtil.readTextFileFromResource("tpch/" + names[i] + ".tbl");
+ String str = JavaResourceUtil.readTextFromResource("tpch/" + names[i] + ".tbl");
Path tablePath = new Path(new Path(tpchTablesDir.toURI()), names[i] + ".tbl");
FileUtil.writeTextToFile(str, tablePath);
paths[i] = tablePath.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
index 6da16ee..eea85fb 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
@@ -188,7 +188,7 @@ public class ExceptionUtil {
}
public static void printStackTraceIfError(Log log, Throwable t) {
- if (!ExceptionUtil.isManagedException(t)) {
+ if (System.getProperty("DEBUG") != null || !ExceptionUtil.isManagedException(t)) {
ExceptionUtil.printStackTrace(log, t);
}
}
@@ -196,4 +196,27 @@ public class ExceptionUtil {
public static UnsupportedException makeNotSupported(String feature) {
return new UnsupportedException(feature);
}
+
+ /**
+ * Return the string about the exception line ; e.g.,)
+ * <code>Line 195 in JdbcTablespace.java</code>
+ *
+ * @return A string representing the line number and source file name at which the exception occurs.
+ */
+ @SuppressWarnings("unused")
+ public static String getExceptionLine() {
+ StackTraceElement stack = Thread.currentThread().getStackTrace()[3];
+ return "Line " + stack.getLineNumber() + " in " + stack.getFileName();
+ }
+
+ /**
+ * Return the string about the exception point; e.g.,)
+ * <code>org.apache.tajo.storage.mysql.JdbcTablespace::createTable</code>
+ *
+ * @return A string representing the class and method names at which the exception occurs.
+ */
+ public static String getExceptionPoint() {
+ StackTraceElement stack = Thread.currentThread().getStackTrace()[3];
+ return stack.getClassName() + "::" + stack.getMethodName();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java b/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
index 3ba6739..95f3a5b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
@@ -25,8 +25,7 @@ public class NotImplementedException extends TajoException {
private static final long serialVersionUID = -5467580471721530536L;
public NotImplementedException() {
- super(Errors.ResultCode.NOT_IMPLEMENTED,
- Thread.currentThread().getStackTrace()[1].getClassName());
+ super(Errors.ResultCode.NOT_IMPLEMENTED, ExceptionUtil.getExceptionPoint());
}
public NotImplementedException(ReturnState state) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
index 6b98d9c..755074b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
@@ -29,7 +29,7 @@ public class UnsupportedException extends TajoException {
}
public UnsupportedException() {
- super(Errors.ResultCode.FEATURE_NOT_SUPPORTED, Thread.currentThread().getStackTrace()[1].getClassName());
+ super(Errors.ResultCode.FEATURE_NOT_SUPPORTED, ExceptionUtil.getExceptionPoint());
}
public UnsupportedException(String featureName) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
index 4e5741b..0c3c031 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
@@ -18,6 +18,8 @@
package org.apache.tajo.storage;
+import org.apache.tajo.exception.UnsupportedException;
+
import javax.annotation.Nullable;
import java.net.URI;
@@ -35,4 +37,6 @@ public interface StorageService {
* @return Table URI
*/
URI getTableURI(@Nullable String spaceName, String databaseName, String tableName);
+
+ long getTableVolumn(URI uri) throws UnsupportedException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
index 2eff9c8..118f42a 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
@@ -18,86 +18,53 @@
package org.apache.tajo.util;
-import com.google.protobuf.Message;
import org.apache.commons.logging.Log;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.tajo.conf.TajoConf;
import java.io.*;
-import java.net.URL;
import java.nio.charset.Charset;
+/**
+ * Simple File Utilities
+ */
public class FileUtil {
- public static void writeProto(File file, Message proto) throws IOException {
- FileOutputStream stream = null;
- try {
- stream = new FileOutputStream(file);
- stream.write(proto.toByteArray());
- } finally {
- IOUtils.closeStream(stream);
- }
- }
-
- public static void writeProto(OutputStream out, Message proto) throws IOException {
- out.write(proto.toByteArray());
- }
-
- public static void writeProto(FileSystem fs, Path path, Message proto) throws IOException {
- FSDataOutputStream stream = fs.create(path);
- try {
- stream.write(proto.toByteArray());
- } finally {
- IOUtils.closeStream(stream);
- }
- }
-
- public static Message loadProto(File file, Message proto) throws IOException {
- FileInputStream in = null;
- try {
- in = new FileInputStream(file);
- Message.Builder builder = proto.newBuilderForType().mergeFrom(in);
- return builder.build();
- } finally {
- IOUtils.closeStream(in);
- }
- }
-
- public static Message loadProto(InputStream in, Message proto) throws IOException {
- Message.Builder builder = proto.newBuilderForType().mergeFrom(in);
- return builder.build();
- }
- public static Message loadProto(FileSystem fs,
- Path path, Message proto) throws IOException {
- FSDataInputStream in = null;
+ public static String readTextFile(File file) throws IOException {
+ StringBuilder fileData = new StringBuilder(1000);
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ char[] buf = new char[1024];
+ int numRead;
try {
- in = new FSDataInputStream(fs.open(path));
- Message.Builder builder = proto.newBuilderForType().mergeFrom(in);
- return builder.build();
+ while ((numRead = reader.read(buf)) != -1) {
+ String readData = String.valueOf(buf, 0, numRead);
+ fileData.append(readData);
+ buf = new char[1024];
+ }
} finally {
- IOUtils.closeStream(in);
+ IOUtils.cleanup(null, reader);
}
- }
-
- public static URL getResourcePath(String resource) throws IOException {
- return ClassLoader.getSystemResource(resource);
+ return fileData.toString();
}
/**
- * It returns a string from a text file found in classpath.
+ * Write a string into a file
*
- * @param resource Resource file name
- * @return String contents if exists. Otherwise, it will return null.
+ * @param text
+ * @param path File path
* @throws IOException
*/
- public static String readTextFileFromResource(String resource) throws IOException {
- InputStream stream = ClassLoader.getSystemResourceAsStream(resource);
- if (stream != null) {
- return readTextFromStream(stream);
- } else {
- throw new FileNotFoundException(resource);
+ public static void writeTextToFile(String text, Path path) throws IOException {
+ FileSystem fs = path.getFileSystem(new TajoConf());
+ if (!fs.exists(path.getParent())) {
+ fs.mkdirs(path.getParent());
}
+ FSDataOutputStream out = fs.create(path);
+ out.write(text.getBytes());
+ out.close();
}
public static String readTextFromStream(InputStream inputStream)
@@ -116,33 +83,6 @@ public class FileUtil {
}
}
- public static String readTextFile(File file) throws IOException {
- StringBuilder fileData = new StringBuilder(1000);
- BufferedReader reader = new BufferedReader(new FileReader(file));
- char[] buf = new char[1024];
- int numRead;
- try {
- while ((numRead = reader.read(buf)) != -1) {
- String readData = String.valueOf(buf, 0, numRead);
- fileData.append(readData);
- buf = new char[1024];
- }
- } finally {
- IOUtils.cleanup(null, reader);
- }
- return fileData.toString();
- }
-
- public static void writeTextToFile(String text, Path path) throws IOException {
- FileSystem fs = path.getFileSystem(new TajoConf());
- if (!fs.exists(path.getParent())) {
- fs.mkdirs(path.getParent());
- }
- FSDataOutputStream out = fs.create(path);
- out.write(text.getBytes());
- out.close();
- }
-
public static void writeTextToStream(String text, OutputStream outputStream) throws IOException {
try {
outputStream.write(text.getBytes());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java
new file mode 100644
index 0000000..a74b000
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+
+/**
+ * An utility for Java resources
+ */
+public class JavaResourceUtil {
+
+ public static URL getResourceURL(String resource) throws IOException {
+ return ClassLoader.getSystemResource(resource);
+ }
+
+ /**
+ * Read a file stored in a local file system and return the string contents.
+ *
+ * @param resource Resource file name
+ * @return String contents
+ * @throws IOException
+ */
+ public static String readTextFromResource(String resource) throws IOException {
+ InputStream stream = ClassLoader.getSystemResourceAsStream(resource);
+ if (stream != null) {
+ return FileUtil.readTextFromStream(stream);
+ } else {
+ throw new FileNotFoundException(resource);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 7a9f0c3..2d57c0a 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -18,6 +18,7 @@
package org.apache.tajo.util;
+import com.google.common.base.Function;
import io.netty.util.CharsetUtil;
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
@@ -405,7 +406,17 @@ public class StringUtils {
* @return A joined string
*/
public static String join(Object[] objects) {
- return join(objects, ", ", 0, objects.length);
+ return join(objects, ", ");
+ }
+
+ /**
+ * Concatenate all objects' string with the delimiter ", "
+ *
+ * @param objects Iterable objects
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter) {
+ return join(objects, delimiter, 0, objects.length);
}
/**
@@ -430,6 +441,28 @@ public class StringUtils {
* @return A joined string
*/
public static String join(Object[] objects, String delimiter, int startIndex, int length) {
+ return join(objects, delimiter, startIndex, length, new Function<Object, String>() {
+ @Override
+ public String apply(Object input) {
+ return input.toString();
+ }
+ });
+ }
+
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param f convert from a type to string
+ * @return A joined string
+ */
+ public static <T> String join(T [] objects, String delimiter, Function<T, String> f) {
+ return join(objects, delimiter, 0, objects.length, f);
+ }
+
+ public static <T> String join(T [] objects, String delimiter, int startIndex, int length, Function<T, String> f) {
boolean first = true;
StringBuilder sb = new StringBuilder();
int endIndex = startIndex + length;
@@ -440,7 +473,7 @@ public class StringUtils {
sb.append(delimiter);
}
- sb.append(objects[i].toString());
+ sb.append(f.apply(objects[i]));
}
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java
new file mode 100644
index 0000000..20c6d2b
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import java.net.URI;
+
+/**
+ * Utility for URI representation
+ */
+public class UriUtil {
+ public static String getScheme(URI uri) {
+ return getScheme(uri.toASCIIString());
+ }
+
+ public static String getScheme(String uri) {
+ return uri.substring(0, uri.indexOf(":/"));
+ }
+
+ /**
+ * Add an query parameter to an existing URI.
+ * @param uri an URI
+ * @param name Parameter name
+ * @param value Parameter value
+ * @return An URI including the given parameter
+ */
+ public static String addParam(String uri, String name, String value) {
+ final String questionMarkOrAnd = uri.split("\\?").length > 1 ? "&" : "?";
+ return uri + questionMarkOrAnd + name + "=" + value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java b/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java
deleted file mode 100644
index 1e7d3c3..0000000
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.util;
-
-import com.google.protobuf.Message;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.apache.tajo.util.TestProtos.TestMessageProto;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestFileUtils {
- private static final String TEST_PATH = "target/test-data/TestFileUTils";
- TestMessageProto proto = null;
-
- @Before
- public void setUp() throws Exception {
- TestMessageProto.Builder builder = TestMessageProto.newBuilder();
- builder.setName("TestFileUtils");
- builder.setAge(30);
- builder.setAddr(TestFileUtils.class.getName());
-
- proto = builder.build();
-
- File testDir = new File(TEST_PATH);
- if(testDir.exists()) {
- testDir.delete();
- }
- testDir.mkdirs();
- }
-
- @After
- public void tearDown() throws Exception {
- File testDir = new File(TEST_PATH);
- if(testDir.exists()) {
- testDir.delete();
- }
- }
-
- @Test
- public final void testWriteLoadProtoFromFile() throws IOException {
- File file = new File(TEST_PATH+"/file.bin");
- file.createNewFile();
- FileUtil.writeProto(file, proto);
-
- Message defaultInstance = TestMessageProto.getDefaultInstance();
- TestMessageProto message = (TestMessageProto)
- FileUtil.loadProto(new File(TEST_PATH+"/file.bin"), defaultInstance);
-
- assertEquals(proto, message);
- }
-
- @Test
- public final void testWriteLoadProtoFromStream() throws IOException {
- FileOutputStream out = new FileOutputStream(new File(TEST_PATH+"/file.bin"));
- FileUtil.writeProto(out, proto);
-
-
- FileInputStream in = new FileInputStream(new File(TEST_PATH+"/file.bin"));
- Message defaultInstance = TestMessageProto.getDefaultInstance();
- TestMessageProto message = (TestMessageProto)
- FileUtil.loadProto(in, defaultInstance);
-
- assertEquals(proto, message);
- }
-
- @Test
- public final void testWriteLoadProtoFromPath() throws IOException {
- Path path = new Path(TEST_PATH+"/file.bin");
- Configuration conf = new Configuration();
- FileSystem localFS = FileSystem.getLocal(conf);
- FileUtil.writeProto(localFS, path, proto);
-
- Message defaultInstance = TestMessageProto.getDefaultInstance();
- TestMessageProto message = (TestMessageProto)
- FileUtil.loadProto(localFS, new Path(TEST_PATH+"/file.bin"),
- defaultInstance);
-
- assertEquals(proto, message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java
new file mode 100644
index 0000000..cefc86f
--- /dev/null
+++ b/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestUriUtil {
+ static final String URI0 = "http://192.168.0.1/table1";
+ static final String URI1 = "hbase:zk://192.168.0.1/table1";
+ static final String URI2 = "jdbc:postgresql://192.168.0.1/table1";
+
+ @Test
+ public void testGetScheme() throws Exception {
+ assertEquals("http", UriUtil.getScheme(URI0));
+ assertEquals("hbase:zk", UriUtil.getScheme(URI1));
+ assertEquals("jdbc:postgresql", UriUtil.getScheme(URI2));
+ }
+
+ @Test
+ public void testAddParam() throws Exception {
+ String userAdded = UriUtil.addParam(URI2, "user", "xxx");
+ assertEquals("jdbc:postgresql://192.168.0.1/table1?user=xxx", userAdded);
+ assertEquals("jdbc:postgresql://192.168.0.1/table1?user=xxx&pass=yyy", UriUtil.addParam(userAdded, "pass", "yyy"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
index 06a54c4..bb8cf1b 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
@@ -25,7 +25,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.storage.StorageConstants;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -61,7 +61,7 @@ public class TestDDLBuilder {
TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod1);
desc.setExternal(true);
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"),
DDLBuilder.buildDDLForExternalTable(desc));
}
@@ -87,20 +87,20 @@ public class TestDDLBuilder {
TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod2);
desc.setExternal(true);
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"),
DDLBuilder.buildDDLForExternalTable(desc));
desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod2);
desc.setExternal(false);
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"),
DDLBuilder.buildDDLForBaseTable(desc));
}
@Test
public void testBuildDDLForBaseTable() throws Exception {
TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1").toUri());
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"),
DDLBuilder.buildDDLForBaseTable(desc));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
index 8642331..760fa62 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
@@ -87,7 +87,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
}
tableSpaceUri = "hbase:zk://" + hostName + ":" + zkPort;
- HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri));
+ HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri), null);
hBaseTablespace.init(new TajoConf(testingCluster.getHBaseUtil().getConf()));
TablespaceManager.addTableSpaceForTest(hBaseTablespace);
}
@@ -484,7 +484,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
new ConstEval(new TextDatum("021")));
scanNode.setQual(evalNodeEq);
Tablespace tablespace = TablespaceManager.getByName("cluster1").get();
- List<Fragment> fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ List<Fragment> fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(1, fragments.size());
assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow()));
assertEquals("021" + postFix, new String(((HBaseFragment)fragments.get(0)).getStopRow()));
@@ -497,7 +497,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2);
scanNode.setQual(evalNodeA);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(2, fragments.size());
HBaseFragment fragment1 = (HBaseFragment) fragments.get(0);
assertEquals("020", new String(fragment1.getStartRow()));
@@ -512,7 +512,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
new ConstEval(new TextDatum("075")));
EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3);
scanNode.setQual(evalNodeB);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(3, fragments.size());
fragment1 = (HBaseFragment) fragments.get(0);
assertEquals("020", new String(fragment1.getStartRow()));
@@ -535,7 +535,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5);
EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC);
scanNode.setQual(evalNodeD);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(3, fragments.size());
fragment1 = (HBaseFragment) fragments.get(0);
@@ -558,7 +558,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5);
evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC);
scanNode.setQual(evalNodeD);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(2, fragments.size());
fragment1 = (HBaseFragment) fragments.get(0);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
index 4dcf562..90a004b 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
@@ -37,7 +37,7 @@ import org.apache.tajo.datum.Int4Datum;
import org.apache.tajo.datum.TextDatum;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.storage.*;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.KeyValueSet;
import org.junit.runners.Parameterized.Parameters;
@@ -248,7 +248,7 @@ public class TestJoinQuery extends QueryTestCaseBase {
TableMeta tableMeta = table.getMeta();
Schema schema = table.getLogicalSchema();
- String[] rows = FileUtil.readTextFileFromResource("tpch/" + tableName + ".tbl").split("\n");
+ String[] rows = JavaResourceUtil.readTextFromResource("tpch/" + tableName + ".tbl").split("\n");
assertTrue(rows.length > 0);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
index f3c302a..3a577e8 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
@@ -33,6 +33,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.SQLSyntaxError;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.Pair;
import org.junit.Rule;
import org.junit.Test;
@@ -143,7 +144,8 @@ public class TestSQLAnalyzer {
String expectedResult = "";
try {
- expectedResult = FileUtil.readTextFileFromResource("results/TestSQLAnalyzer/errors/" + fileName + ".result");
+ expectedResult =
+ JavaResourceUtil.readTextFromResource("results/TestSQLAnalyzer/errors/" + fileName + ".result");
} catch (FileNotFoundException fnfe) {
}
@@ -182,7 +184,7 @@ public class TestSQLAnalyzer {
String fileName = null;
try {
fileName = pair.getFirst().split("\\.")[0];
- expectedResult = FileUtil.readTextFileFromResource("results/TestSQLAnalyzer/" + fileName + ".result");
+ expectedResult = JavaResourceUtil.readTextFromResource("results/TestSQLAnalyzer/" + fileName + ".result");
} catch (FileNotFoundException ioe) {
expectedResult = "";
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
index e45dd75..c1d5a97 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
@@ -74,13 +74,10 @@ public class TestRowFile {
FileTablespace sm = (FileTablespace) TablespaceManager.get(cluster.getDefaultFileSystem().getUri()).get();
Path tablePath = new Path("/test");
- Path metaPath = new Path(tablePath, ".meta");
Path dataPath = new Path(tablePath, "test.tbl");
FileSystem fs = sm.getFileSystem();
fs.mkdirs(tablePath);
- FileUtil.writeProto(fs, metaPath, meta.getProto());
-
Appender appender = sm.getAppender(meta, schema, dataPath);
appender.enableStats();
appender.init();
@@ -105,14 +102,11 @@ public class TestRowFile {
assertEquals(tupleNum, stat.getNumRows().longValue());
FileStatus file = fs.getFileStatus(dataPath);
- TableProto proto = (TableProto) FileUtil.loadProto(
- cluster.getDefaultFileSystem(), metaPath, TableProto.getDefaultInstance());
- meta = new TableMeta(proto);
FileFragment fragment = new FileFragment("test.tbl", dataPath, 0, file.getLen());
int tupleCnt = 0;
start = System.currentTimeMillis();
- Scanner scanner = sm.getScanner(meta, schema, fragment);
+ Scanner scanner = sm.getScanner(meta, schema, fragment, null);
scanner.init();
while ((tuple=scanner.next()) != null) {
tupleCnt++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
index cc740e7..8b87ea9 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
@@ -130,7 +130,7 @@ public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext>
}
}
- public EvalNode visitUnaryEval(EvalCodeGenContext context, Stack<EvalNode> stack, UnaryEval unary) {
+ public EvalNode visitUnaryEval(EvalCodeGenContext context, UnaryEval unary, Stack<EvalNode> stack) {
stack.push(unary);
if (unary.getType() == EvalType.CAST) {
visitCast(context, stack, (CastEval) unary);
@@ -349,7 +349,7 @@ public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext>
return cast;
}
- public EvalNode visitField(EvalCodeGenContext context, Stack<EvalNode> stack, FieldEval field) {
+ public EvalNode visitField(EvalCodeGenContext context, FieldEval field, Stack<EvalNode> stack) {
if (field.getValueType().getType() == TajoDataTypes.Type.NULL_TYPE) {
context.pushNullOfThreeValuedLogic();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
index 63b5445..c2a6e99 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
@@ -174,9 +174,6 @@ public class SeqScanExec extends ScanExec {
scanIt = new FilterScanIterator(scanner, qual);
} else {
- if (scanner.isSelectable()) { // TODO - isSelectable should be moved to FormatProperty
- scanner.setFilter(qual);
- }
scanIt = new FullScanIterator(scanner);
}
}
@@ -275,10 +272,17 @@ public class SeqScanExec extends ScanExec {
this.scanner = tablespace.getScanner(
meta,
plan.getPhysicalSchema(),
- fragments[0],
+ FragmentConvertor.convert(context.getConf(), fragments[0]),
projected);
}
+ if (scanner.isSelectable()) { // TODO - isSelectable should be moved to FormatProperty
+ scanner.setFilter(qual);
+ }
+
+ if (plan.hasLimit()) {
+ scanner.setLimit(plan.getLimit());
+ }
scanner.init();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index ed7698f..326bc86 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -19,6 +19,7 @@
package org.apache.tajo.master;
import com.codahale.metrics.Gauge;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -58,6 +59,7 @@ import org.apache.tajo.rule.SelfDiagnosisRuleSession;
import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.service.ServiceTrackerFactory;
import org.apache.tajo.session.SessionManager;
+import org.apache.tajo.storage.TablespaceManager;
import org.apache.tajo.util.*;
import org.apache.tajo.util.history.HistoryReader;
import org.apache.tajo.util.history.HistoryWriter;
@@ -180,11 +182,11 @@ public class TajoMaster extends CompositeService {
this.dispatcher = new AsyncDispatcher();
addIfService(dispatcher);
- // check the system directory and create if they are not created.
- checkAndInitializeSystemDirectories();
- diagnoseTajoMaster();
+ // check the system directory and create if they are not created.
+ checkAndInitializeSystemDirectories();
+ diagnoseTajoMaster();
- catalogServer = new CatalogServer(Collections.EMPTY_SET, loadFunctions());
+ catalogServer = new CatalogServer(TablespaceManager.getMetadataProviders(), loadFunctions());
addIfService(catalogServer);
catalog = new LocalCatalogWrapper(catalogServer, systemConf);
@@ -409,6 +411,14 @@ public class TajoMaster extends CompositeService {
LOG.info("Tajo Master main thread exiting");
}
+ /**
+ * This is only for unit tests.
+ */
+ @VisibleForTesting
+ public void refresh() {
+ catalogServer.refresh(TablespaceManager.getMetadataProviders());
+ }
+
public EventHandler getEventHandler() {
return dispatcher.getEventHandler();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
index 809b81f..ac40c4f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
@@ -109,7 +109,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc
FileTablespace fileTablespace = TUtil.checkTypeAndGet(tablespace, FileTablespace.class);
fragments.addAll(Repartitioner.getFragmentsFromPartitionedTable(fileTablespace, scanNode, tableDesc));
} else {
- fragments.addAll(tablespace.getSplits(tableDesc.getName(), tableDesc, scanNode));
+ fragments.addAll(tablespace.getSplits(tableDesc.getName(), tableDesc, scanNode.getQual()));
}
if (!fragments.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
index f7aff30..ffdb3f0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
@@ -291,6 +291,7 @@ public class QueryExecutor {
if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
maxRow = (int) limitNode.getFetchFirstNum();
+ scanNode.setLimit(maxRow);
}
if (desc.getStats().getNumRows() == 0) {
desc.getStats().setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
index c4fc645..3ce86b1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
@@ -19,6 +19,7 @@
package org.apache.tajo.querymaster;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -113,7 +114,7 @@ public class Repartitioner {
// So, we need to handle FileFragment by its size.
// If we don't check its size, it can cause IndexOutOfBoundsException.
Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
- List<Fragment> fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc);
+ List<Fragment> fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc, null);
if (fileFragments.size() > 0) {
fragments[i] = fileFragments.get(0);
} else {
@@ -389,8 +390,8 @@ public class Repartitioner {
} else {
- Collection<Fragment> scanFragments = space.getSplits(eachScan.getCanonicalName(),
- tableDesc, eachScan);
+ Collection<Fragment> scanFragments =
+ space.getSplits(eachScan.getCanonicalName(), tableDesc, eachScan.getQual());
if (scanFragments != null) {
rightFragments.addAll(scanFragments);
}
@@ -460,22 +461,24 @@ public class Repartitioner {
/**
* It creates a number of fragments for all partitions.
*/
- public static List<Fragment> getFragmentsFromPartitionedTable(FileTablespace sm,
+ public static List<Fragment> getFragmentsFromPartitionedTable(Tablespace tsHandler,
ScanNode scan,
TableDesc table) throws IOException {
+ Preconditions.checkArgument(tsHandler instanceof FileTablespace, "tsHandler must be FileTablespace");
if (!(scan instanceof PartitionedTableScanNode)) {
throw new IllegalArgumentException("scan should be a PartitionedTableScanNode type.");
}
List<Fragment> fragments = Lists.newArrayList();
PartitionedTableScanNode partitionsScan = (PartitionedTableScanNode) scan;
- fragments.addAll(sm.getSplits(
+ fragments.addAll(((FileTablespace) tsHandler).getSplits(
scan.getCanonicalName(), table.getMeta(), table.getSchema(), partitionsScan.getInputPaths()));
partitionsScan.setInputPaths(null);
return fragments;
}
private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext schedulerContext, Stage stage,
- int baseScanId, Fragment[] fragments) throws IOException {
+ int baseScanId, Fragment[] fragments)
+ throws IOException, TajoException {
ExecutionBlock execBlock = stage.getBlock();
ScanNode[] scans = execBlock.getScanNodes();
@@ -499,20 +502,20 @@ public class Repartitioner {
for (int i = 0; i < scans.length; i++) {
ScanNode scan = scans[i];
TableDesc desc = stage.getContext().getTableDesc(scan);
- TableMeta meta = desc.getMeta();
Collection<Fragment> scanFragments;
Path[] partitionScanPaths = null;
- FileTablespace space = (FileTablespace) TablespaceManager.get(desc.getUri()).get();
+
+ Tablespace space = TablespaceManager.get(desc.getUri()).get();
if (scan.getType() == NodeType.PARTITIONS_SCAN) {
- PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)scan;
+ PartitionedTableScanNode partitionScan = (PartitionedTableScanNode) scan;
partitionScanPaths = partitionScan.getInputPaths();
// set null to inputPaths in getFragmentsFromPartitionedTable()
scanFragments = getFragmentsFromPartitionedTable(space, scan, desc);
} else {
- scanFragments = space.getSplits(scan.getCanonicalName(), desc, scan);
+ scanFragments = space.getSplits(scan.getCanonicalName(), desc, scan.getQual());
}
if (scanFragments != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 68916e4..125c83c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -1144,7 +1144,7 @@ public class Stage implements EventHandler<StageEvent> {
// After calling this method, partition paths are removed from the physical plan.
fragments = Repartitioner.getFragmentsFromPartitionedTable((FileTablespace) tablespace, scan, table);
} else {
- fragments = tablespace.getSplits(scan.getCanonicalName(), table, scan);
+ fragments = tablespace.getSplits(scan.getCanonicalName(), table, scan.getQual());
}
Stage.scheduleFragments(stage, fragments);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml
index fdce467..b0e9f64 100644
--- a/tajo-dist/pom.xml
+++ b/tajo-dist/pom.xml
@@ -143,7 +143,8 @@
run cp -r $ROOT/tajo-catalog/target/tajo-catalog-${project.version}/* .
run cp -r $ROOT/tajo-storage/target/tajo-storage-${project.version}/* .
run cp -r $ROOT/tajo-sql-parser/target/tajo-sql-parser-${project.version}/* .
-
+ run cp -r $ROOT/tajo-storage/tajo-storage-jdbc/target/tajo-storage-jdbc-${project.version}.jar .
+ run cp -r $ROOT/tajo-storage/tajo-storage-pgsql/target/tajo-storage-pgsql-${project.version}.jar .
run cp -r $ROOT/tajo-pullserver/target/tajo-pullserver-${project.version}.jar .
run cp -r $ROOT/tajo-metrics/target/tajo-metrics-${project.version}.jar .
run cp -r $ROOT/tajo-core/target/tajo-core-${project.version}.jar .
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/index.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst
index ec65fd3..5a54809 100644
--- a/tajo-docs/src/main/sphinx/index.rst
+++ b/tajo-docs/src/main/sphinx/index.rst
@@ -37,7 +37,7 @@ Table of Contents:
functions
table_management
table_partitioning
- storage_plugin
+ storage_plugins
index_overview
backup_and_restore
hive_integration
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugin.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugin.rst b/tajo-docs/src/main/sphinx/storage_plugin.rst
deleted file mode 100644
index d9c6838..0000000
--- a/tajo-docs/src/main/sphinx/storage_plugin.rst
+++ /dev/null
@@ -1,47 +0,0 @@
-*************************************
-Storage Plugin
-*************************************
-
-Overview
-========
-
-Tajo supports various storage systems, such as HDFS, Amazon S3, Openstack Swift, and HBase. Also, we have a plan to support RDBMS storages like Oracle, MySQL, PostgreSQL. Tajo already embeds HDFS, S3, Openstack, and HBase, and also Tajo allows users to register custom storages and data formats to Tajo cluster instances. This section describes how you register custom storages and data types.
-
-Register custom storage
-=======================
-
-First of all, your storage implementation should be packed as a jar file. Then, please copy the jar file into ``tajo/extlib`` directory. Next, you should copy ``conf/storage-site.json.template`` into ``conf/storage-site.json`` and modify the file like the below.
-
-Configuration
-=============
-
-Tajo has a default configuration for builtin storages, such as HDFS, local file system, and Amazon S3. it also allows users to add custom storage plugins
-
-``conf/storage-site.json`` file has the following struct:
-
-.. code-block:: json
-
- {
- "storages": {
- "${scheme}": {
- "handler": "${class name}"
- }
- }
- }
-
-Each storage instance (i.e., :doc:`/table_management/tablespaces`) is identified by an URI. The scheme of URI plays a role to identify storage type. For example, ``hdfs://`` is used for Hdfs storage, ``jdbc://`` is used for JDBC-based storage, and ``hbase://`` is used for HBase storage.
-
-You should substitute a scheme name without ``://`` for ``${scheme}``.
-
-See an example for HBase storage.
-
-.. code-block:: json
-
- {
- "storages": {
- "hbase": {
- "handler": "org.apache.tajo.storage.hbase.HBaseTablespace",
- "default-format": "hbase"
- }
- }
- }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugins.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugins.rst b/tajo-docs/src/main/sphinx/storage_plugins.rst
new file mode 100644
index 0000000..35b3d38
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/storage_plugins.rst
@@ -0,0 +1,11 @@
+*************************************
+Storage Plugin
+*************************************
+
+This section describes the storage plugins available in Tajo to access datasets from different data sources.
+
+.. toctree::
+ :maxdepth: 1
+
+ storage_plugins/overview
+ storage_plugins/postgresql
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugins/overview.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugins/overview.rst b/tajo-docs/src/main/sphinx/storage_plugins/overview.rst
new file mode 100644
index 0000000..243d144
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/storage_plugins/overview.rst
@@ -0,0 +1,47 @@
+*************************************
+Storage Plugin Overview
+*************************************
+
+Overview
+========
+
+Tajo supports various storage systems, such as HDFS, Amazon S3, Openstack Swift, HBase, and RDBMS. Tajo already embeds HDFS, S3, Openstack, HBase, RDBMS storage plugins, and also Tajo allows users to register custom storages and data formats to Tajo cluster instances. This section describes how you register custom storages and data types.
+
+Register custom storage
+=======================
+
+First of all, your storage implementation should be packed as a jar file. Then, please copy the jar file into ``tajo/extlib`` directory. Next, you should copy ``conf/storage-site.json.template`` into ``conf/storage-site.json`` and modify the file like the below.
+
+Configuration
+=============
+
+Tajo has a default configuration for builtin storages, such as HDFS, local file system, and Amazon S3. it also allows users to add custom storage plugins
+
+``conf/storage-site.json`` file has the following struct:
+
+.. code-block:: json
+
+ {
+ "storages": {
+ "${scheme}": {
+ "handler": "${class name}"
+ }
+ }
+ }
+
+Each storage instance (i.e., :doc:`/table_management/tablespaces`) is identified by an URI. The scheme of URI plays a role to identify storage type. For example, ``hdfs://`` is used for Hdfs storage, ``jdbc://`` is used for JDBC-based storage, and ``hbase://`` is used for HBase storage.
+
+You should substitute a scheme name without ``://`` for ``${scheme}``.
+
+See an example for HBase storage.
+
+.. code-block:: json
+
+ {
+ "storages": {
+ "hbase": {
+ "handler": "org.apache.tajo.storage.hbase.HBaseTablespace",
+ "default-format": "hbase"
+ }
+ }
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst b/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst
new file mode 100644
index 0000000..9c9a22b
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst
@@ -0,0 +1,40 @@
+*************************************
+PostgreSQL Storage Handler
+*************************************
+
+Overview
+========
+
+PostgreSQL storage handler is available by default in Tajo. It enables users' queries to access database objects in PostgreSQL. Tables in PostgreSQL will be shown as tables in Tajo too. Most of the SQL queries used for PostgreSQL are available in Tajo via this storage handles. Its main advantages is to allow federated query processing among tables in stored HDFS and PostgreSQL.
+
+Configuration
+=============
+
+PostgreSQL storage handler is a builtin storage handler. So, you can eaisly register PostgreSQL databases to a Tajo cluster if you just add the following line to ``conf/storage-site.json`` file. If you want to know more information about ``storage-site.json``, please refer to :doc:`/table_management/tablespaces`.
+
+.. code-block:: json
+
+ {
+ "spaces": {
+ "pgsql_db1": {
+ "uri": "jdbc:postgresql://hostname:port/db1"
+
+ "configs": {
+ "mapped_database": "tajo_db1"
+ "connection_properties": {
+ "user": "tajo",
+ "password": "xxxx"
+ }
+ }
+ }
+ }
+ }
+
+``configs`` allows users to specific additional configurations.
+``mapped_database`` specifies a database name shown in Tajo. In the example, the database ``db1`` in PostgreSQL
+will be mapped to the database ``tajo_db1`` in Tajo.
+``connection_properties`` allows users to set JDBC connection parameters.
+Please refer to https://jdbc.postgresql.org/documentation/head/connect.html in order to know the details of
+PostgreSQL connection parameters.
+
+The storage-site.json will be effective after you restart a tajo cluster.
\ No newline at end of file