You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2015/09/02 13:06:26 UTC
[03/39] tajo git commit: Implement the base jdbc tablespace and its
unit test base.
Implement the base jdbc tablespace and its unit test base.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/920e8a42
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/920e8a42
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/920e8a42
Branch: refs/heads/TAJO-1730
Commit: 920e8a42689e3f38d1ccac7eff3b8a60dd0a8a46
Parents: 5ca1a4a
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sun Aug 9 23:36:24 2015 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sun Aug 9 23:36:24 2015 +0900
----------------------------------------------------------------------
.../apache/tajo/catalog/MetadataProvider.java | 6 +-
.../tajo/catalog/LinkedMetadataManager.java | 2 +-
.../tajo/catalog/TestLinkedMetadataManager.java | 12 +-
.../apache/tajo/storage/StorageProperty.java | 15 +-
.../org/apache/tajo/storage/Tablespace.java | 9 +-
.../tajo/storage/hbase/HBaseTablespace.java | 3 +-
.../org/apache/tajo/storage/FileTablespace.java | 2 +-
tajo-storage/tajo-storage-jdbc/pom.xml | 22 +-
.../storage/jdbc/JdbcMetadataProviderBase.java | 244 +++++++++++++++++++
.../tajo/storage/jdbc/JdbcTablespace.java | 21 +-
.../tajo/storage/hbase/TestJdbcTableSpace.java | 72 ------
.../tajo/storage/jdbc/TestJdbcTableSpace.java | 98 ++++++++
.../storage/jdbc/TestMysqlJdbcTableSpace.java | 95 ++++++++
.../storage/jdbc/TestPgSQLJdbcTableSpace.java | 87 +++++++
14 files changed, 589 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
index 216d33c..4d55f6f 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
@@ -31,9 +31,9 @@ public interface MetadataProvider {
String getDatabaseName();
- Collection<String> getCatalogs();
+ Collection<String> getSchemas();
- Collection<String> getTables(@Nullable String catalog);
+ Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern);
- TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException;
+ TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
index 080c984..dc438e8 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
@@ -148,7 +148,7 @@ public class LinkedMetadataManager {
public Collection<String> getSchemas(@Nullable String dbName) throws UndefinedDatabaseException {
ensureIfDBExists(dbName);
- return providerMap.get(dbName).getCatalogs();
+ return providerMap.get(dbName).getSchemas();
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
index 4ddf7ab..95550b6 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
@@ -89,17 +89,17 @@ public class TestLinkedMetadataManager {
}
@Override
- public Collection<String> getCatalogs() {
+ public Collection<String> getSchemas() {
return Lists.newArrayList("cat1", "cat2");
}
@Override
- public Collection<String> getTables(@Nullable String catalog) {
+ public Collection<String> getTables(@Nullable String schemaPattern) {
return Lists.newArrayList("table1", "table2");
}
@Override
- public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException {
+ public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException {
if (tableName.equals("table1")) {
return TABLE1;
} else if (tableName.equals("table2")) {
@@ -128,17 +128,17 @@ public class TestLinkedMetadataManager {
}
@Override
- public Collection<String> getCatalogs() {
+ public Collection<String> getSchemas() {
return Lists.newArrayList("cat3", "cat4");
}
@Override
- public Collection<String> getTables(@Nullable String catalog) {
+ public Collection<String> getTables(@Nullable String schemaPattern) {
return Lists.newArrayList("table3", "table4");
}
@Override
- public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException {
+ public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException {
if (tableName.equals("table3")) {
return TABLE3;
} else if (tableName.equals("table4")) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
index c1db34e..41ecd38 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageProperty.java
@@ -30,16 +30,20 @@ public class StorageProperty {
private final boolean writable;
/** if this storage allows use of artibrary paths */
private final boolean absolutePathAllowed;
+ /** if this storage provides metadata provider */
+ private final boolean metadataProvided;
public StorageProperty(String defaultFormat,
boolean movable,
boolean writable,
- boolean absolutePathAllowed) {
+ boolean absolutePathAllowed,
+ boolean metadataProvided) {
this.defaultFormat = defaultFormat;
this.movable = movable;
this.writable = writable;
this.absolutePathAllowed = absolutePathAllowed;
+ this.metadataProvided = metadataProvided;
}
/**
@@ -76,4 +80,13 @@ public class StorageProperty {
public boolean isArbitraryPathAllowed() {
return this.absolutePathAllowed;
}
+
+ /**
+ * Is metadata provided?
+ *
+ * @return True if this storage provides linked metadata.
+ */
+ public boolean isMetadataProvided() {
+ return this.metadataProvided;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
index 968601c..9da48f4 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
@@ -22,10 +22,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.TajoException;
@@ -424,4 +421,8 @@ public abstract class Tablespace {
TableMeta meta) throws IOException {
throw new IOException("Staging the output result is not supported in this storage");
}
+
+ public MetadataProvider getMetadataProvider() {
+ throw new UnsupportedException("Linked Metadata Provider for " + name);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
index 35c974b..c15d91f 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
@@ -66,7 +66,8 @@ import java.util.*;
public class HBaseTablespace extends Tablespace {
private static final Log LOG = LogFactory.getLog(HBaseTablespace.class);
- public static final StorageProperty HBASE_STORAGE_PROPERTIES = new StorageProperty("hbase", false, true, false);
+ public static final StorageProperty HBASE_STORAGE_PROPERTIES =
+ new StorageProperty("hbase", false, true, false, false);
public static final FormatProperty HFILE_FORMAT_PROPERTIES = new FormatProperty(true, false, true);
public static final FormatProperty PUT_MODE_PROPERTIES = new FormatProperty(true, true, false);
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
index e8a6c12..b6e58b8 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
@@ -95,7 +95,7 @@ public class FileTablespace extends Tablespace {
}
};
- private static final StorageProperty FileStorageProperties = new StorageProperty("TEXT", true, true, true);
+ private static final StorageProperty FileStorageProperties = new StorageProperty("TEXT", true, true, true, false);
private static final FormatProperty GeneralFileProperties = new FormatProperty(true, false, true);
protected FileSystem fs;
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml
index 93c15bb..40978ea 100644
--- a/tajo-storage/tajo-storage-jdbc/pom.xml
+++ b/tajo-storage/tajo-storage-jdbc/pom.xml
@@ -53,8 +53,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
<encoding>${project.build.sourceEncoding}</encoding>
</configuration>
</plugin>
@@ -306,6 +306,24 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>testing-mysql-server</artifactId>
+ <version>0.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.airlift</groupId>
+ <artifactId>testing-postgresql-server</artifactId>
+ <version>0.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>15.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java
new file mode 100644
index 0000000..1658b46
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.facebook.presto.hive.shaded.com.google.common.base.Function;
+import com.facebook.presto.hive.shaded.com.google.common.collect.Collections2;
+import com.facebook.presto.hive.shaded.com.google.common.collect.Lists;
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.exception.UndefinedTablespaceException;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.common.TajoDataTypes.DataType;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.Pair;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+import java.sql.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType;
+
+public abstract class JdbcMetadataProviderBase implements MetadataProvider {
+ protected static final Log LOG = LogFactory.getLog(JdbcMetadataProviderBase.class);
+
+ protected final JdbcTablespace space;
+ protected final String databaseName;
+
+ protected final String jdbcUri;
+ protected final String username;
+ protected final String password;
+
+ protected final Connection connection;
+
+ public JdbcMetadataProviderBase(JdbcTablespace space, String dbName) {
+ this.space = space;
+ this.databaseName = dbName;
+
+ this.jdbcUri = "";
+ this.username = "";
+ this.password = "";
+
+ String jdbcDriver = getJdbcDriverName();
+ try {
+ Class.forName(jdbcDriver).newInstance();
+ LOG.info("Loaded JDBC driver (" + jdbcDriver + ")");
+ } catch (Exception e) {
+ throw new TajoInternalError(e);
+ }
+
+ try {
+ connection = DriverManager.getConnection(jdbcUri, this.username, this.password);
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ }
+ }
+
+ @Override
+ public String getTablespaceName() {
+ return space.getName();
+ }
+
+ @Override
+ public URI getTablespaceUri() {
+ return space.getUri();
+ }
+
+ @Override
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ @Override
+ public Collection<String> getSchemas() {
+ return Collections.EMPTY_SET;
+ }
+
+ @Override
+ public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) {
+ ResultSet res = null;
+ List<String> tableNames = Lists.newArrayList();
+ try {
+ res = connection.getMetaData().getTables(databaseName, schemaPattern, tablePattern, new String [] {});
+ while(res.next()) {
+ tableNames.add(res.getString("TABLE_NAME"));
+ }
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ } finally {
+ try {
+ if (res != null) {
+ res.close();
+ }
+ } catch (SQLException e) {
+ LOG.warn(e);
+ }
+ }
+
+ return tableNames;
+ }
+
+ private TypeDesc convertDataType(ResultSet res) throws SQLException {
+ final int typeId = res.getInt("DATA_TYPE");
+
+ switch (typeId ) {
+ case Types.BOOLEAN:
+ return new TypeDesc(newSimpleDataType(Type.BOOLEAN));
+
+ case Types.TINYINT:
+ case Types.SMALLINT:
+ case Types.INTEGER:
+ return new TypeDesc(newSimpleDataType(Type.INT4));
+
+ case Types.BIGINT:
+ return new TypeDesc(newSimpleDataType(Type.INT8));
+
+ case Types.FLOAT:
+ return new TypeDesc(newSimpleDataType(Type.FLOAT4));
+
+ case Types.DOUBLE:
+ return new TypeDesc(newSimpleDataType(Type.FLOAT8));
+
+ case Types.DATE:
+ return new TypeDesc(newSimpleDataType(Type.DATE));
+
+ case Types.TIME:
+ return new TypeDesc(newSimpleDataType(Type.TIME));
+
+ case Types.TIMESTAMP:
+ return new TypeDesc(newSimpleDataType(Type.TIMESTAMP));
+
+ case Types.CHAR:
+ case Types.NCHAR:
+ case Types.VARCHAR:
+ case Types.NVARCHAR:
+ case Types.CLOB:
+ case Types.NCLOB:
+ case Types.LONGVARCHAR:
+ case Types.LONGNVARCHAR:
+ new TypeDesc(newSimpleDataType(Type.TEXT));
+
+ case Types.BINARY:
+ case Types.VARBINARY:
+ case Types.BLOB:
+ new TypeDesc(newSimpleDataType(Type.BLOB));
+
+ default:
+ throw new UnsupportedException("DATA_TYPE(" + typeId + ")");
+ }
+ }
+
+ @Override
+ public TableDesc getTableDescriptor(String schemaName, String tableName) throws UndefinedTablespaceException {
+ ResultSet resultForTable = null;
+ ResultSet resultForColumns = null;
+ try {
+
+ // get table name
+ resultForTable = connection.getMetaData().getTables(databaseName, schemaName, tableName, new String[]{});
+ final String name = resultForTable.getString("TABLE_NAME");
+ Preconditions.checkState(tableName.equals(name));
+
+ // get columns
+ resultForColumns = connection.getMetaData().getColumns(databaseName, schemaName, tableName, null);
+
+ List<Pair<Integer, Column>> columns = Lists.newArrayList();
+
+ while(resultForColumns.next()) {
+ final int ordinalPos = resultForColumns.getInt("ORDINAL_POSITION");
+ final String qualifier = resultForColumns.getString("TABLE_NAME");
+ final String columnName = resultForColumns.getString("COLUMN_NAME");
+ final TypeDesc type = convertDataType(resultForColumns);
+ Column c = new Column(CatalogUtil.buildFQName(qualifier, columnName), type);
+
+ columns.add(new Pair<Integer, Column>(ordinalPos, c));
+ }
+
+ // sort columns in an order of ordinal position
+ Collections.sort(columns, new Comparator<Pair<Integer, Column>>() {
+ @Override
+ public int compare(Pair<Integer, Column> o1, Pair<Integer, Column> o2) {
+ return o1.getFirst() - o2.getFirst();
+ }
+ });
+
+ // transform the pair list into collection for columns
+ Schema schema = new Schema(Collections2.transform(columns, new Function<Pair<Integer, Column>, Column>() {
+ @Override
+ public Column apply(@Nullable Pair<Integer, Column> columnPair) {
+ return columnPair.getSecond();
+ }
+ }));
+
+ return new TableDesc(
+ CatalogUtil.buildFQName(databaseName, name),
+ schema,
+ new TableMeta("rowstore", new KeyValueSet()),
+ space.getTableUri(databaseName, name)
+ );
+
+ } catch (SQLException e) {
+ throw new TajoInternalError(e);
+ } finally {
+ try {
+ if (resultForTable != null) {
+ resultForTable.close();
+ }
+
+ if (resultForColumns != null) {
+ resultForColumns.close();
+ }
+
+ } catch (SQLException e) {
+ LOG.warn(e);
+ }
+ }
+ }
+
+ protected abstract String getJdbcDriverName();
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
index 95fd258..0cb13fc 100644
--- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
@@ -21,10 +21,8 @@ package org.apache.tajo.storage.jdbc;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.exception.UnsupportedException;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.ScanNode;
@@ -47,6 +45,8 @@ import java.util.Map;
*/
public class JdbcTablespace extends Tablespace {
+ static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false, true);
+
public JdbcTablespace(String name, URI uri) {
super(name, uri);
}
@@ -86,8 +86,6 @@ public class JdbcTablespace extends Tablespace {
return null;
}
- static final StorageProperty STORAGE_PROPERTY = new StorageProperty("rowstore", false, true, false);
-
@Override
public StorageProperty getProperty() {
return STORAGE_PROPERTY;
@@ -104,7 +102,11 @@ public class JdbcTablespace extends Tablespace {
}
@Override
- public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException {
+ public TupleRange[] getInsertSortRanges(OverridableConf queryContext,
+ TableDesc tableDesc,
+ Schema inputSchema,
+ SortSpec[] sortSpecs,
+ TupleRange dataRange) throws IOException {
return new TupleRange[0];
}
@@ -115,7 +117,6 @@ public class JdbcTablespace extends Tablespace {
@Override
public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException {
-
}
@Override
@@ -143,4 +144,8 @@ public class JdbcTablespace extends Tablespace {
public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException {
return null;
}
+
+ public MetadataProvider getMetadataProvider() {
+ throw new UnsupportedException("Linked Metadata Provider for " + name);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java
deleted file mode 100644
index f489e47..0000000
--- a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/hbase/TestJdbcTableSpace.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.hbase;
-
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.TextDatum;
-import org.apache.tajo.plan.expr.*;
-import org.apache.tajo.plan.logical.ScanNode;
-import org.apache.tajo.storage.TablespaceManager;
-import org.apache.tajo.storage.jdbc.JdbcTablespace;
-import org.apache.tajo.util.Pair;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-public class TestJdbcTableSpace {
- @BeforeClass
- public static void setUp() throws IOException {
- String mysqlUri = "jdbc:mysql://host1:2171/db1";
- JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri));
- mysqlTablespace.init(new TajoConf());
- TablespaceManager.addTableSpaceForTest(mysqlTablespace);
-
- String pgsqlUri = "jdbc:postgres://host1:2615/db2";
- JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri));
- pgSQLTablespace.init(new TajoConf());
- TablespaceManager.addTableSpaceForTest(pgSQLTablespace);
- }
-
- @Test
- public void testTablespaceHandler() throws Exception {
- assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace);
- assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName()));
- assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace);
- assertEquals(URI.create("jdbc:mysql://host1:2171/db1"),
- TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri());
-
- assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace);
- assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName()));
- assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace);
-
- assertEquals(URI.create("jdbc:postgres://host1:2615/db2"),
- TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri());
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java
new file mode 100644
index 0000000..997a419
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestJdbcTableSpace.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.common.collect.ImmutableSet;
+import io.airlift.testing.mysql.TestingMySqlServer;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.TablespaceManager;
+import org.apache.tajo.storage.jdbc.JdbcTablespace;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestJdbcTableSpace {
+ @BeforeClass
+ public static void setUp() throws IOException {
+ String mysqlUri = "jdbc:mysql://host1:2171/db1";
+ JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri));
+ mysqlTablespace.init(new TajoConf());
+ TablespaceManager.addTableSpaceForTest(mysqlTablespace);
+
+ String pgsqlUri = "jdbc:postgres://host1:2615/db2";
+ JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri));
+ pgSQLTablespace.init(new TajoConf());
+ TablespaceManager.addTableSpaceForTest(pgSQLTablespace);
+ }
+
+ @Test
+ public void testTablespaceHandler() throws Exception {
+ assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace);
+ assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName()));
+ assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace);
+ assertEquals(URI.create("jdbc:mysql://host1:2171/db1"),
+ TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri());
+
+ assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace);
+ assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName()));
+ assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace);
+
+ assertEquals(URI.create("jdbc:postgres://host1:2615/db2"),
+ TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri());
+ }
+
+ @Test
+ public void test() throws Exception {
+ try (TestingMySqlServer server = new TestingMySqlServer("testuser", "testpass", "db1", "db2")) {
+ assertTrue(server.isRunning());
+ assertTrue(server.isReadyForConnections());
+ assertEquals(server.getMySqlVersion(), "5.5.9");
+ assertEquals(server.getDatabases(), ImmutableSet.of("db1", "db2"));
+ assertEquals(server.getUser(), "testuser");
+ assertEquals(server.getPassword(), "testpass");
+ assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:");
+ assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort());
+
+ for (String database : server.getDatabases()) {
+ try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) {
+ connection.setCatalog(database);
+ try (Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)");
+ statement.execute("INSERT INTO test_table (c1) VALUES (1)");
+ try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) {
+ assertTrue(resultSet.next());
+ assertEquals(resultSet.getLong(1), 1L);
+ assertFalse(resultSet.next());
+ }
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java
new file mode 100644
index 0000000..966fa8b
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestMysqlJdbcTableSpace.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import com.google.common.collect.ImmutableSet;
+import io.airlift.testing.mysql.TestingMySqlServer;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.TablespaceManager;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.junit.Assert.*;
+
+public class TestMysqlJdbcTableSpace {
+ @BeforeClass
+ public static void setUp() throws IOException {
+ String mysqlUri = "jdbc:mysql://host1:2171/db1";
+ JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri));
+ mysqlTablespace.init(new TajoConf());
+ TablespaceManager.addTableSpaceForTest(mysqlTablespace);
+
+ String pgsqlUri = "jdbc:postgres://host1:2615/db2";
+ JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri));
+ pgSQLTablespace.init(new TajoConf());
+ TablespaceManager.addTableSpaceForTest(pgSQLTablespace);
+ }
+
+ @Test
+ public void testTablespaceHandler() throws Exception {
+ assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace);
+ assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName()));
+ assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace);
+ assertEquals(URI.create("jdbc:mysql://host1:2171/db1"),
+ TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri());
+
+ assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace);
+ assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName()));
+ assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace);
+
+ assertEquals(URI.create("jdbc:postgres://host1:2615/db2"),
+ TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri());
+ }
+
+ @Test
+ public void test() throws Exception {
+ try (TestingMySqlServer server = new TestingMySqlServer("testuser", "testpass", "db1", "db2")) {
+ assertTrue(server.isRunning());
+ assertTrue(server.isReadyForConnections());
+ assertEquals(server.getMySqlVersion(), "5.5.9");
+ assertEquals(server.getDatabases(), ImmutableSet.of("db1", "db2"));
+ assertEquals(server.getUser(), "testuser");
+ assertEquals(server.getPassword(), "testpass");
+ assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:");
+ assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort());
+
+ for (String database : server.getDatabases()) {
+ try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) {
+ connection.setCatalog(database);
+ try (Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)");
+ statement.execute("INSERT INTO test_table (c1) VALUES (1)");
+ try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) {
+ assertTrue(resultSet.next());
+ assertEquals(resultSet.getLong(1), 1L);
+ assertFalse(resultSet.next());
+ }
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/920e8a42/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java
new file mode 100644
index 0000000..aef70a9
--- /dev/null
+++ b/tajo-storage/tajo-storage-jdbc/src/test/java/org/apache/tajo/storage/jdbc/TestPgSQLJdbcTableSpace.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.jdbc;
+
+import io.airlift.testing.postgresql.TestingPostgreSqlServer;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.TablespaceManager;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.junit.Assert.*;
+
+public class TestPgSQLJdbcTableSpace {
+ @BeforeClass
+ public static void setUp() throws IOException {
+ String mysqlUri = "jdbc:mysql://host1:2171/db1";
+ JdbcTablespace mysqlTablespace = new JdbcTablespace("cluster2", URI.create(mysqlUri));
+ mysqlTablespace.init(new TajoConf());
+ TablespaceManager.addTableSpaceForTest(mysqlTablespace);
+
+ String pgsqlUri = "jdbc:postgres://host1:2615/db2";
+ JdbcTablespace pgSQLTablespace = new JdbcTablespace("cluster3", URI.create(pgsqlUri));
+ pgSQLTablespace.init(new TajoConf());
+ TablespaceManager.addTableSpaceForTest(pgSQLTablespace);
+ }
+
+ @Test
+ public void testTablespaceHandler() throws Exception {
+ assertTrue((TablespaceManager.getByName("cluster2").get()) instanceof JdbcTablespace);
+ assertEquals("cluster2", (TablespaceManager.getByName("cluster2").get().getName()));
+ assertTrue((TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get()) instanceof JdbcTablespace);
+ assertEquals(URI.create("jdbc:mysql://host1:2171/db1"),
+ TablespaceManager.get(URI.create("jdbc:mysql://host1:2171/db1")).get().getUri());
+
+ assertTrue((TablespaceManager.getByName("cluster3").get()) instanceof JdbcTablespace);
+ assertEquals("cluster3", (TablespaceManager.getByName("cluster3").get().getName()));
+ assertTrue((TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get()) instanceof JdbcTablespace);
+
+ assertEquals(URI.create("jdbc:postgres://host1:2615/db2"),
+ TablespaceManager.get(URI.create("jdbc:postgres://host1:2615/db2")).get().getUri());
+ }
+
+ @Test
+ public void test() throws Exception {
+ try (TestingPostgreSqlServer server = new TestingPostgreSqlServer("testuser", "testdb")) {
+ assertEquals(server.getUser(), "testuser");
+ assertEquals(server.getDatabase(), "testdb");
+ assertEquals(server.getJdbcUrl().substring(0, 5), "jdbc:");
+ assertEquals(server.getPort(), URI.create(server.getJdbcUrl().substring(5)).getPort());
+
+ try (Connection connection = DriverManager.getConnection(server.getJdbcUrl())) {
+ try (Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE test_table (c1 bigint PRIMARY KEY)");
+ statement.execute("INSERT INTO test_table (c1) VALUES (1)");
+ try (ResultSet resultSet = statement.executeQuery("SELECT count(*) FROM test_table")) {
+ assertTrue(resultSet.next());
+ assertEquals(resultSet.getLong(1), 1L);
+ assertFalse(resultSet.next());
+ }
+ }
+ }
+ }
+ }
+}