You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2015/09/17 14:42:27 UTC
[4/4] tajo git commit: TAJO-1730: JDBC Tablespace support.
TAJO-1730: JDBC Tablespace support.
Closes #719
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0e4ad563
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0e4ad563
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0e4ad563
Branch: refs/heads/master
Commit: 0e4ad5635be05bdde8dac5feb6c9a85fb3cf0e06
Parents: e1c2d35
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Sep 17 05:40:58 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Sep 17 05:40:58 2015 -0700
----------------------------------------------------------------------
CHANGES | 2 +
.../apache/tajo/catalog/MetadataProvider.java | 40 ++-
.../java/org/apache/tajo/catalog/TableUtil.java | 51 ---
.../apache/tajo/catalog/TestFunctionDesc.java | 19 --
.../org/apache/tajo/catalog/CatalogServer.java | 15 +-
.../tajo/catalog/LinkedMetadataManager.java | 23 +-
.../tajo/catalog/store/AbstractDBStore.java | 9 +-
.../tajo/catalog/TestLinkedMetadataManager.java | 12 +-
.../tajo/client/CatalogAdminClientImpl.java | 8 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 17 +-
.../org/apache/tajo/TajoTestingCluster.java | 2 +-
.../test/java/org/apache/tajo/TpchTestBase.java | 4 +-
.../apache/tajo/exception/ExceptionUtil.java | 25 +-
.../tajo/exception/NotImplementedException.java | 3 +-
.../tajo/exception/UnsupportedException.java | 2 +-
.../org/apache/tajo/storage/StorageService.java | 4 +
.../java/org/apache/tajo/util/FileUtil.java | 116 ++-----
.../org/apache/tajo/util/JavaResourceUtil.java | 50 +++
.../java/org/apache/tajo/util/StringUtils.java | 37 +-
.../main/java/org/apache/tajo/util/UriUtil.java | 46 +++
.../org/apache/tajo/util/TestFileUtils.java | 107 ------
.../java/org/apache/tajo/util/TestUriUtil.java | 43 +++
.../apache/tajo/cli/tools/TestDDLBuilder.java | 10 +-
.../tajo/engine/query/TestHBaseTable.java | 12 +-
.../apache/tajo/engine/query/TestJoinQuery.java | 4 +-
.../apache/tajo/parser/sql/TestSQLAnalyzer.java | 6 +-
.../org/apache/tajo/storage/TestRowFile.java | 8 +-
.../tajo/engine/codegen/EvalCodeGenerator.java | 4 +-
.../engine/planner/physical/SeqScanExec.java | 12 +-
.../java/org/apache/tajo/master/TajoMaster.java | 18 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../apache/tajo/master/exec/QueryExecutor.java | 1 +
.../apache/tajo/querymaster/Repartitioner.java | 23 +-
.../java/org/apache/tajo/querymaster/Stage.java | 2 +-
tajo-dist/pom.xml | 3 +-
tajo-docs/src/main/sphinx/index.rst | 2 +-
tajo-docs/src/main/sphinx/storage_plugin.rst | 47 ---
tajo-docs/src/main/sphinx/storage_plugins.rst | 11 +
.../main/sphinx/storage_plugins/overview.rst | 47 +++
.../main/sphinx/storage_plugins/postgresql.rst | 40 +++
.../org/apache/tajo/plan/LogicalPlanner.java | 28 +-
.../apache/tajo/plan/expr/AlgebraicUtil.java | 2 +-
.../org/apache/tajo/plan/expr/EvalType.java | 14 +-
.../tajo/plan/expr/SimpleEvalNodeVisitor.java | 13 +-
.../plan/exprrewrite/rules/ConstantFolding.java | 2 +-
.../exprrewrite/rules/ConstantPropagation.java | 2 +-
.../org/apache/tajo/plan/logical/ScanNode.java | 30 +-
.../rules/CommonConditionReduceRule.java | 2 +-
.../tajo/plan/serder/EvalNodeSerializer.java | 6 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 44 ++-
tajo-plan/src/main/proto/Plan.proto | 8 +-
tajo-project/pom.xml | 11 +
tajo-storage/pom.xml | 2 +
.../apache/tajo/storage/AbstractScanner.java | 10 +
.../org/apache/tajo/storage/MergeScanner.java | 10 +
.../org/apache/tajo/storage/NullScanner.java | 11 +
.../apache/tajo/storage/OldStorageManager.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 16 +
.../apache/tajo/storage/StorageProperty.java | 15 +-
.../org/apache/tajo/storage/Tablespace.java | 98 +++---
.../apache/tajo/storage/TablespaceManager.java | 84 +++--
.../storage/fragment/FragmentConvertor.java | 5 +-
.../src/main/resources/storage-default.json | 6 +-
.../src/main/resources/storage-default.xml | 8 +-
.../src/test/resources/storage-default.xml | 4 +
.../apache/tajo/storage/hbase/HBaseScanner.java | 10 +
.../tajo/storage/hbase/HBaseTablespace.java | 49 ++-
.../tajo/storage/hbase/TestHBaseTableSpace.java | 11 +-
.../org/apache/tajo/storage/FileScanner.java | 12 +
.../org/apache/tajo/storage/FileTablespace.java | 41 ++-
.../tajo/storage/text/CSVLineDeserializer.java | 8 +-
.../tajo/storage/TestDelimitedTextFile.java | 12 +-
.../apache/tajo/storage/TestFileTablespace.java | 9 +-
.../org/apache/tajo/storage/TestLineReader.java | 3 +-
.../org/apache/tajo/storage/TestStorages.java | 76 ++---
.../apache/tajo/storage/avro/TestAvroUtil.java | 3 +-
.../apache/tajo/storage/json/TestJsonSerDe.java | 4 +-
tajo-storage/tajo-storage-jdbc/pom.xml | 243 ++++++++++++++
.../tajo/storage/jdbc/ConnectionInfo.java | 115 +++++++
.../apache/tajo/storage/jdbc/JdbcFragment.java | 106 ++++++
.../storage/jdbc/JdbcMetadataProviderBase.java | 252 ++++++++++++++
.../apache/tajo/storage/jdbc/JdbcScanner.java | 334 +++++++++++++++++++
.../tajo/storage/jdbc/JdbcTablespace.java | 209 ++++++++++++
.../apache/tajo/storage/jdbc/SQLBuilder.java | 188 +++++++++++
.../storage/jdbc/SQLExpressionGenerator.java | 302 +++++++++++++++++
.../src/main/proto/JdbcFragmentProtos.proto | 31 ++
.../tajo/storage/jdbc/TestConnectionInfo.java | 52 +++
tajo-storage/tajo-storage-pgsql/pom.xml | 257 ++++++++++++++
.../tajo/storage/pgsql/PgSQLJdbcScanner.java | 38 +++
.../storage/pgsql/PgSQLMetadataProvider.java | 41 +++
.../tajo/storage/pgsql/PgSQLTablespace.java | 71 ++++
.../tajo/storage/pgsql/PgSQLTestServer.java | 211 ++++++++++++
.../storage/pgsql/TestPgSQLEndPointTests.java | 74 ++++
.../storage/pgsql/TestPgSQLJdbcTableSpace.java | 127 +++++++
.../pgsql/TestPgSQLMetadataProvider.java | 84 +++++
.../tajo/storage/pgsql/TestPgSQLQueryTests.java | 200 +++++++++++
.../pgsql/TestPgSQLSimpleQueryTests.java | 48 +++
.../src/test/resources/dataset/.marker | 1 +
.../src/test/resources/pgsql/customer.sql | 10 +
.../src/test/resources/pgsql/datetime_types.sql | 5 +
.../src/test/resources/pgsql/datetime_types.txt | 4 +
.../src/test/resources/pgsql/lineitem.sql | 18 +
.../src/test/resources/pgsql/nation.sql | 6 +
.../src/test/resources/pgsql/orders.sql | 11 +
.../src/test/resources/pgsql/part.sql | 11 +
.../src/test/resources/pgsql/partsupp.sql | 7 +
.../src/test/resources/pgsql/region.sql | 5 +
.../src/test/resources/pgsql/supplier.sql | 9 +
.../TestPgSQLQueryTests/testBetweenDates.sql | 1 +
.../TestPgSQLQueryTests/testBetweenNumbers.sql | 1 +
.../TestPgSQLQueryTests/testCaseWhenFilter.sql | 6 +
.../TestPgSQLQueryTests/testCountAsterisk.sql | 1 +
.../TestPgSQLQueryTests/testCtasToHdfs.sql | 1 +
.../TestPgSQLQueryTests/testDateTimeTypes.sql | 1 +
.../testFixedLengthFields.sql | 1 +
.../testFunctionWithinFilter.sql | 1 +
.../testInPredicateWithLiterals.sql | 1 +
.../testInPredicateWithNumbers.sql | 1 +
.../testProjectedColumns.sql | 1 +
.../testQueryWithConnProperties.sql | 1 +
.../TestPgSQLQueryTests/testSimpleFilter.sql | 1 +
.../queries/TestPgSQLQueryTests/testTPCH_Q1.sql | 21 ++
.../TestPgSQLQueryTests/testTPCH_Q2_Part.sql | 18 +
.../testTPCH_Q2_Part_MixedStorage.sql | 18 +
.../queries/TestPgSQLQueryTests/testTPCH_Q3.sql | 22 ++
.../queries/TestPgSQLQueryTests/testTPCH_Q5.sql | 25 ++
.../testVariableLengthFields.sql | 1 +
.../TestPgSQLSimpleQueryTests/testSelectAll.sql | 1 +
.../testSelectLimit.sql | 1 +
.../TestPgSQLQueryTests/testBetweenDates.result | 5 +
.../testBetweenNumbers.result | 5 +
.../testCaseWhenFilter.result | 5 +
.../testCountAsterisk.result | 3 +
.../TestPgSQLQueryTests/testCtasToHdfs.result | 7 +
.../testDateTimeTypes.result | 6 +
.../testFixedLengthFields.result | 5 +
.../testFunctionWithinFilter.result | 5 +
.../testInPredicateWithLiterals.result | 5 +
.../testInPredicateWithNumbers.result | 5 +
.../testProjectedColumns.result | 5 +
.../testQueryWithConnProperties.result | 5 +
.../TestPgSQLQueryTests/testSimpleFilter.result | 5 +
.../TestPgSQLQueryTests/testTPCH_Q1.result | 4 +
.../TestPgSQLQueryTests/testTPCH_Q2_Part.result | 3 +
.../testTPCH_Q2_Part_MixedStorage.result | 3 +
.../TestPgSQLQueryTests/testTPCH_Q3.result | 5 +
.../TestPgSQLQueryTests/testTPCH_Q5.result | 27 ++
.../testVariableLengthFields.result | 5 +
.../testSelectAll.result | 7 +
.../testSelectLimit.result | 5 +
150 files changed, 4200 insertions(+), 678 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 122f895..0c201f1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,8 @@ Release 0.11.0 - unreleased
NEW FEATURES
+ TAJO-1730: JDBC Tablespace support. (hyunsik)
+
TAJO-1812: Timezone support in JSON file format. (hyunsik)
TAJO-1486: Text file should support to skip header rows when creating
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
index cafe0a1..f0d0e39 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/MetadataProvider.java
@@ -25,15 +25,49 @@ import java.net.URI;
import java.util.Collection;
public interface MetadataProvider {
+ /**
+ * Get a Tablespace name
+ *
+ * @return Tablespace name
+ */
String getTablespaceName();
+ /**
+ * Get a tablespace URI
+ *
+ * @return Table space URI
+ */
URI getTablespaceUri();
+ /**
+ * Return a database name
+ *
+ * @return Database name
+ */
String getDatabaseName();
- Collection<String> getCatalogs();
+ /**
+ * Get a list of schema names
+ * @return All schema names in this database
+ */
+ Collection<String> getSchemas();
- Collection<String> getTables(@Nullable String catalog);
+ /**
+ * get Table names matched to a given patterns
+ *
+ * @param schemaPattern Schema name pattern
+ * @param tablePattern Table name pattern
+ * @return All matched table names
+ */
+ Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern);
- TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException;
+ /**
+ * Get a table descriptor
+ *
+ * @param schemaName schema name
+ * @param tableName table name
+ * @return TableDesc
+ * @throws UndefinedTablespaceException
+ */
+ TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
deleted file mode 100644
index 76aec33..0000000
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableUtil.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.catalog;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.util.FileUtil;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-public class TableUtil {
- public static TableMeta getTableMeta(Configuration conf, Path tablePath)
- throws IOException {
- TableMeta meta = null;
-
- FileSystem fs = tablePath.getFileSystem(conf);
-
- Path tableMetaPath = new Path(tablePath, ".meta");
- if(!fs.exists(tableMetaPath)) {
- throw new FileNotFoundException(".meta file not found in "+tablePath.toString());
- }
- FSDataInputStream tableMetaIn =
- fs.open(tableMetaPath);
-
- TableProto tableProto = (TableProto) FileUtil.loadProto(tableMetaIn,
- TableProto.getDefaultInstance());
- meta = new TableMeta(tableProto);
-
- return meta;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
index 16e85f0..e8170ee 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java
@@ -71,25 +71,6 @@ public class TestFunctionDesc {
assertEquals(Type.INT4, desc.getReturnType().getType());
assertArrayEquals(CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8),
desc.getParamTypes());
-
- CommonTestingUtil.getTestDir(TEST_PATH);
- File save = new File(TEST_PATH + "/save.dat");
- FileUtil.writeProto(save, desc.getProto());
-
- FunctionDescProto proto = FunctionDescProto.getDefaultInstance();
- proto = (FunctionDescProto) FileUtil.loadProto(save, proto);
-
- FunctionDesc newDesc = new FunctionDesc(proto);
-
- assertEquals("sum", newDesc.getFunctionName());
- assertEquals(TestSum.class, newDesc.getLegacyFuncClass());
- assertEquals(FunctionType.GENERAL, newDesc.getFuncType());
- assertEquals(Type.INT4, newDesc.getReturnType().getType());
-
- assertArrayEquals(CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8),
- newDesc.getParamTypes());
-
- assertEquals(desc.getProto(), newDesc.getProto());
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index dff292f..8f4e0e8 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -18,6 +18,7 @@
package org.apache.tajo.catalog;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
@@ -84,7 +85,7 @@ public class CatalogServer extends AbstractService {
private Map<String, List<FunctionDescProto>> functions = new ConcurrentHashMap<String,
List<FunctionDescProto>>();
- protected final LinkedMetadataManager linkedMetadataManager;
+ protected LinkedMetadataManager linkedMetadataManager;
protected final InfoSchemaMetadataDictionary metaDictionary = new InfoSchemaMetadataDictionary();
// RPC variables
@@ -102,7 +103,8 @@ public class CatalogServer extends AbstractService {
this.builtingFuncs = new ArrayList<FunctionDesc>();
}
- public CatalogServer(Set<MetadataProvider> metadataProviders, Collection<FunctionDesc> sqlFuncs) throws IOException {
+ public CatalogServer(Collection<MetadataProvider> metadataProviders, Collection<FunctionDesc> sqlFuncs)
+ throws IOException {
super(CatalogServer.class.getName());
this.handler = new CatalogProtocolHandler();
this.linkedMetadataManager = new LinkedMetadataManager(metadataProviders);
@@ -192,6 +194,15 @@ public class CatalogServer extends AbstractService {
super.serviceStop();
}
+ /**
+ * Refresh the linked metadata manager. This must be used for only testing.
+ * @param metadataProviders
+ */
+ @VisibleForTesting
+ public void refresh(Collection<MetadataProvider> metadataProviders) {
+ this.linkedMetadataManager = new LinkedMetadataManager(metadataProviders);
+ }
+
public CatalogProtocolHandler getHandler() {
return this.handler;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
index 6d4af1f..a4c718e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java
@@ -32,7 +32,6 @@ import javax.annotation.Nullable;
import java.net.URI;
import java.util.Collection;
import java.util.Map;
-import java.util.regex.Pattern;
import static com.google.common.collect.Collections2.filter;
@@ -163,7 +162,7 @@ public class LinkedMetadataManager {
public Collection<String> getSchemas(@Nullable String dbName) throws UndefinedDatabaseException {
ensureIfDBExists(dbName);
- return providerMap.get(dbName).getCatalogs();
+ return providerMap.get(dbName).getSchemas();
}
/**
@@ -180,21 +179,9 @@ public class LinkedMetadataManager {
*/
public Collection<String> getTableNames(String dbName,
@Nullable final String schemaPattern,
- final String tablePattern) throws UndefinedDatabaseException {
+ @Nullable final String tablePattern) throws UndefinedDatabaseException {
ensureIfDBExists(dbName);
-
- if (tablePattern == null) { // all tables in this database
- return providerMap.get(dbName).getTables("null");
-
- } else {
- final Pattern pattern = Pattern.compile(tablePattern);
- return filter(providerMap.get(dbName).getTables(schemaPattern), new Predicate<String>() {
- @Override
- public boolean apply(@Nullable String input) {
- return pattern.matcher(tablePattern).matches();
- }
- });
- }
+ return providerMap.get(dbName).getTables(schemaPattern, tablePattern);
}
/**
@@ -232,7 +219,7 @@ public class LinkedMetadataManager {
*/
public boolean existsTable(String dbName, String schemaName, String tableName) {
if (providerMap.containsKey(dbName)) {
- return providerMap.get(dbName).getTables(schemaName).contains(tableName);
+ return providerMap.get(dbName).getTables(schemaName, tableName).contains(tableName);
}
return false;
@@ -251,6 +238,6 @@ public class LinkedMetadataManager {
ensureIfDBExists(dbName);
- return providerMap.get(dbName).getTableDescriptor(schemaName, tbName);
+ return providerMap.get(dbName).getTableDesc(schemaName, tbName);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index 0b1b120..0f07149 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -33,14 +33,17 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.*;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import java.io.IOException;
import java.net.URI;
import java.sql.*;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto;
@@ -198,7 +201,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
public String readSchemaFile(String path) {
try {
- return FileUtil.readTextFileFromResource("schemas/" + path);
+ return JavaResourceUtil.readTextFromResource("schemas/" + path);
} catch (IOException e) {
throw new TajoInternalError(e);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
index 30c563f..a065fa8 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestLinkedMetadataManager.java
@@ -82,17 +82,17 @@ public class TestLinkedMetadataManager {
}
@Override
- public Collection<String> getCatalogs() {
+ public Collection<String> getSchemas() {
return Lists.newArrayList("cat1", "cat2");
}
@Override
- public Collection<String> getTables(@Nullable String catalog) {
+ public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) {
return Lists.newArrayList("table1", "table2");
}
@Override
- public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException {
+ public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException {
if (tableName.equals("table1")) {
return TABLE1;
} else if (tableName.equals("table2")) {
@@ -121,17 +121,17 @@ public class TestLinkedMetadataManager {
}
@Override
- public Collection<String> getCatalogs() {
+ public Collection<String> getSchemas() {
return Lists.newArrayList("cat3", "cat4");
}
@Override
- public Collection<String> getTables(@Nullable String catalog) {
+ public Collection<String> getTables(@Nullable String schemaPattern, @Nullable String tablePattern) {
return Lists.newArrayList("table3", "table4");
}
@Override
- public TableDesc getTableDescriptor(String catalogName, String tableName) throws UndefinedTablespaceException {
+ public TableDesc getTableDesc(String schemaName, String tableName) throws UndefinedTablespaceException {
if (tableName.equals("table3")) {
return TABLE3;
} else if (tableName.equals("table4")) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
index fb13f07..102b39e 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
@@ -87,15 +87,15 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
}
@Override
- public void dropDatabase(final String databaseName) throws UndefinedDatabaseException {
+ public void dropDatabase(final String databaseName)
+ throws UndefinedDatabaseException, InsufficientPrivilegeException {
try {
final BlockingInterface stub = conn.getTMStub();
final ReturnState state = stub.dropDatabase(null, conn.getSessionedString(databaseName));
- if (isThisError(state, ResultCode.UNDEFINED_DATABASE)) {
- throw new UndefinedDatabaseException(state);
- }
+ throwsIfThisError(state, UndefinedDatabaseException.class);
+ throwsIfThisError(state, InsufficientPrivilegeException.class);
ensureOk(state);
} catch (ServiceException e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index 5a912e5..9abfbd3 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -19,6 +19,7 @@
package org.apache.tajo;
import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.protobuf.ServiceException;
@@ -39,6 +40,7 @@ import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.exception.InsufficientPrivilegeException;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.UndefinedTableException;
import org.apache.tajo.jdbc.FetchResultSet;
@@ -206,10 +208,13 @@ public class QueryTestCaseBase {
client = testBase.getTestingCluster().newTajoClient();
URL datasetBaseURL = ClassLoader.getSystemResource("dataset");
+ Preconditions.checkNotNull(datasetBaseURL, "dataset directory is absent.");
datasetBasePath = new Path(datasetBaseURL.toString());
URL queryBaseURL = ClassLoader.getSystemResource("queries");
+ Preconditions.checkNotNull(queryBaseURL, "queries directory is absent.");
queryBasePath = new Path(queryBaseURL.toString());
URL resultBaseURL = ClassLoader.getSystemResource("results");
+ Preconditions.checkNotNull(resultBaseURL, "results directory is absent.");
resultBasePath = new Path(resultBaseURL.toString());
}
@@ -223,11 +228,19 @@ public class QueryTestCaseBase {
// if the current database is "default", shouldn't drop it.
if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
for (String tableName : catalog.getAllTableNames(currentDatabase)) {
- client.updateQuery("DROP TABLE IF EXISTS " + tableName);
+ try {
+ client.updateQuery("DROP TABLE IF EXISTS " + tableName);
+ } catch (InsufficientPrivilegeException i) {
+ LOG.warn("relation '" + tableName + "' is read only.");
+ }
}
client.selectDatabase(TajoConstants.DEFAULT_DATABASE_NAME);
- client.dropDatabase(currentDatabase);
+ try {
+ client.dropDatabase(currentDatabase);
+ } catch (InsufficientPrivilegeException e) {
+ LOG.warn("database '" + currentDatabase + "' is read only.");
+ }
}
client.close();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 680b5ec..8d2ee8f 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -345,7 +345,7 @@ public class TajoTestingCluster {
URI defaultTsUri = TajoConf.getWarehouseDir(c).toUri();
FileTablespace defaultTableSpace =
- new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri);
+ new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri, null);
defaultTableSpace.init(conf);
TablespaceManager.addTableSpaceForTest(defaultTableSpace);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
index 027e735..62f7477 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
@@ -27,10 +27,10 @@ import org.apache.tajo.catalog.Schema;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.KeyValueSet;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.ResultSet;
import java.util.Map;
@@ -76,7 +76,7 @@ public class TpchTestBase {
File tpchTablesDir = new File(new File(CommonTestingUtil.getTestDir().toUri()), "tpch");
for (int i = 0; i < names.length; i++) {
- String str = FileUtil.readTextFileFromResource("tpch/" + names[i] + ".tbl");
+ String str = JavaResourceUtil.readTextFromResource("tpch/" + names[i] + ".tbl");
Path tablePath = new Path(new Path(tpchTablesDir.toURI()), names[i] + ".tbl");
FileUtil.writeTextToFile(str, tablePath);
paths[i] = tablePath.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
index 6da16ee..eea85fb 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/ExceptionUtil.java
@@ -188,7 +188,7 @@ public class ExceptionUtil {
}
public static void printStackTraceIfError(Log log, Throwable t) {
- if (!ExceptionUtil.isManagedException(t)) {
+ if (System.getProperty("DEBUG") != null || !ExceptionUtil.isManagedException(t)) {
ExceptionUtil.printStackTrace(log, t);
}
}
@@ -196,4 +196,27 @@ public class ExceptionUtil {
public static UnsupportedException makeNotSupported(String feature) {
return new UnsupportedException(feature);
}
+
+ /**
+ * Return the string about the exception line ; e.g.,)
+ * <code>Line 195 in JdbcTablespace.java</code>
+ *
+ * @return A string representing the line number and source file name at which the exception occurs.
+ */
+ @SuppressWarnings("unused")
+ public static String getExceptionLine() {
+ StackTraceElement stack = Thread.currentThread().getStackTrace()[3];
+ return "Line " + stack.getLineNumber() + " in " + stack.getFileName();
+ }
+
+ /**
+ * Return the string about the exception point; e.g.,)
+ * <code>org.apache.tajo.storage.mysql.JdbcTablespace::createTable</code>
+ *
+ * @return A string representing the class and method names at which the exception occurs.
+ */
+ public static String getExceptionPoint() {
+ StackTraceElement stack = Thread.currentThread().getStackTrace()[3];
+ return stack.getClassName() + "::" + stack.getMethodName();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java b/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
index 3ba6739..95f3a5b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/NotImplementedException.java
@@ -25,8 +25,7 @@ public class NotImplementedException extends TajoException {
private static final long serialVersionUID = -5467580471721530536L;
public NotImplementedException() {
- super(Errors.ResultCode.NOT_IMPLEMENTED,
- Thread.currentThread().getStackTrace()[1].getClassName());
+ super(Errors.ResultCode.NOT_IMPLEMENTED, ExceptionUtil.getExceptionPoint());
}
public NotImplementedException(ReturnState state) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
index 6b98d9c..755074b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
+++ b/tajo-common/src/main/java/org/apache/tajo/exception/UnsupportedException.java
@@ -29,7 +29,7 @@ public class UnsupportedException extends TajoException {
}
public UnsupportedException() {
- super(Errors.ResultCode.FEATURE_NOT_SUPPORTED, Thread.currentThread().getStackTrace()[1].getClassName());
+ super(Errors.ResultCode.FEATURE_NOT_SUPPORTED, ExceptionUtil.getExceptionPoint());
}
public UnsupportedException(String featureName) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
index 4e5741b..0c3c031 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
@@ -18,6 +18,8 @@
package org.apache.tajo.storage;
+import org.apache.tajo.exception.UnsupportedException;
+
import javax.annotation.Nullable;
import java.net.URI;
@@ -35,4 +37,6 @@ public interface StorageService {
* @return Table URI
*/
URI getTableURI(@Nullable String spaceName, String databaseName, String tableName);
+
+ long getTableVolumn(URI uri) throws UnsupportedException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
index 2eff9c8..118f42a 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/FileUtil.java
@@ -18,86 +18,53 @@
package org.apache.tajo.util;
-import com.google.protobuf.Message;
import org.apache.commons.logging.Log;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.tajo.conf.TajoConf;
import java.io.*;
-import java.net.URL;
import java.nio.charset.Charset;
+/**
+ * Simple File Utilities
+ */
public class FileUtil {
- public static void writeProto(File file, Message proto) throws IOException {
- FileOutputStream stream = null;
- try {
- stream = new FileOutputStream(file);
- stream.write(proto.toByteArray());
- } finally {
- IOUtils.closeStream(stream);
- }
- }
-
- public static void writeProto(OutputStream out, Message proto) throws IOException {
- out.write(proto.toByteArray());
- }
-
- public static void writeProto(FileSystem fs, Path path, Message proto) throws IOException {
- FSDataOutputStream stream = fs.create(path);
- try {
- stream.write(proto.toByteArray());
- } finally {
- IOUtils.closeStream(stream);
- }
- }
-
- public static Message loadProto(File file, Message proto) throws IOException {
- FileInputStream in = null;
- try {
- in = new FileInputStream(file);
- Message.Builder builder = proto.newBuilderForType().mergeFrom(in);
- return builder.build();
- } finally {
- IOUtils.closeStream(in);
- }
- }
-
- public static Message loadProto(InputStream in, Message proto) throws IOException {
- Message.Builder builder = proto.newBuilderForType().mergeFrom(in);
- return builder.build();
- }
- public static Message loadProto(FileSystem fs,
- Path path, Message proto) throws IOException {
- FSDataInputStream in = null;
+ public static String readTextFile(File file) throws IOException {
+ StringBuilder fileData = new StringBuilder(1000);
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ char[] buf = new char[1024];
+ int numRead;
try {
- in = new FSDataInputStream(fs.open(path));
- Message.Builder builder = proto.newBuilderForType().mergeFrom(in);
- return builder.build();
+ while ((numRead = reader.read(buf)) != -1) {
+ String readData = String.valueOf(buf, 0, numRead);
+ fileData.append(readData);
+ buf = new char[1024];
+ }
} finally {
- IOUtils.closeStream(in);
+ IOUtils.cleanup(null, reader);
}
- }
-
- public static URL getResourcePath(String resource) throws IOException {
- return ClassLoader.getSystemResource(resource);
+ return fileData.toString();
}
/**
- * It returns a string from a text file found in classpath.
+ * Write a string into a file
*
- * @param resource Resource file name
- * @return String contents if exists. Otherwise, it will return null.
+ * @param text
+ * @param path File path
* @throws IOException
*/
- public static String readTextFileFromResource(String resource) throws IOException {
- InputStream stream = ClassLoader.getSystemResourceAsStream(resource);
- if (stream != null) {
- return readTextFromStream(stream);
- } else {
- throw new FileNotFoundException(resource);
+ public static void writeTextToFile(String text, Path path) throws IOException {
+ FileSystem fs = path.getFileSystem(new TajoConf());
+ if (!fs.exists(path.getParent())) {
+ fs.mkdirs(path.getParent());
}
+ FSDataOutputStream out = fs.create(path);
+ out.write(text.getBytes());
+ out.close();
}
public static String readTextFromStream(InputStream inputStream)
@@ -116,33 +83,6 @@ public class FileUtil {
}
}
- public static String readTextFile(File file) throws IOException {
- StringBuilder fileData = new StringBuilder(1000);
- BufferedReader reader = new BufferedReader(new FileReader(file));
- char[] buf = new char[1024];
- int numRead;
- try {
- while ((numRead = reader.read(buf)) != -1) {
- String readData = String.valueOf(buf, 0, numRead);
- fileData.append(readData);
- buf = new char[1024];
- }
- } finally {
- IOUtils.cleanup(null, reader);
- }
- return fileData.toString();
- }
-
- public static void writeTextToFile(String text, Path path) throws IOException {
- FileSystem fs = path.getFileSystem(new TajoConf());
- if (!fs.exists(path.getParent())) {
- fs.mkdirs(path.getParent());
- }
- FSDataOutputStream out = fs.create(path);
- out.write(text.getBytes());
- out.close();
- }
-
public static void writeTextToStream(String text, OutputStream outputStream) throws IOException {
try {
outputStream.write(text.getBytes());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java
new file mode 100644
index 0000000..a74b000
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/util/JavaResourceUtil.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+
+/**
+ * An utility for Java resources
+ */
+public class JavaResourceUtil {
+
+ public static URL getResourceURL(String resource) throws IOException {
+ return ClassLoader.getSystemResource(resource);
+ }
+
+ /**
+ * Read a file stored in a local file system and return the string contents.
+ *
+ * @param resource Resource file name
+ * @return String contents
+ * @throws IOException
+ */
+ public static String readTextFromResource(String resource) throws IOException {
+ InputStream stream = ClassLoader.getSystemResourceAsStream(resource);
+ if (stream != null) {
+ return FileUtil.readTextFromStream(stream);
+ } else {
+ throw new FileNotFoundException(resource);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 7a9f0c3..2d57c0a 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -18,6 +18,7 @@
package org.apache.tajo.util;
+import com.google.common.base.Function;
import io.netty.util.CharsetUtil;
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
@@ -405,7 +406,17 @@ public class StringUtils {
* @return A joined string
*/
public static String join(Object[] objects) {
- return join(objects, ", ", 0, objects.length);
+ return join(objects, ", ");
+ }
+
+ /**
+ * Concatenate all objects' string with the delimiter ", "
+ *
+ * @param objects Iterable objects
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter) {
+ return join(objects, delimiter, 0, objects.length);
}
/**
@@ -430,6 +441,28 @@ public class StringUtils {
* @return A joined string
*/
public static String join(Object[] objects, String delimiter, int startIndex, int length) {
+ return join(objects, delimiter, startIndex, length, new Function<Object, String>() {
+ @Override
+ public String apply(Object input) {
+ return input.toString();
+ }
+ });
+ }
+
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param f convert from a type to string
+ * @return A joined string
+ */
+ public static <T> String join(T [] objects, String delimiter, Function<T, String> f) {
+ return join(objects, delimiter, 0, objects.length, f);
+ }
+
+ public static <T> String join(T [] objects, String delimiter, int startIndex, int length, Function<T, String> f) {
boolean first = true;
StringBuilder sb = new StringBuilder();
int endIndex = startIndex + length;
@@ -440,7 +473,7 @@ public class StringUtils {
sb.append(delimiter);
}
- sb.append(objects[i].toString());
+ sb.append(f.apply(objects[i]));
}
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java
new file mode 100644
index 0000000..20c6d2b
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/util/UriUtil.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import java.net.URI;
+
+/**
+ * Utility for URI representation
+ */
+public class UriUtil {
+ public static String getScheme(URI uri) {
+ return getScheme(uri.toASCIIString());
+ }
+
+ public static String getScheme(String uri) {
+ return uri.substring(0, uri.indexOf(":/"));
+ }
+
+ /**
+ * Add an query parameter to an existing URI.
+ * @param uri an URI
+ * @param name Parameter name
+ * @param value Parameter value
+ * @return An URI including the given parameter
+ */
+ public static String addParam(String uri, String name, String value) {
+ final String questionMarkOrAnd = uri.split("\\?").length > 1 ? "&" : "?";
+ return uri + questionMarkOrAnd + name + "=" + value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java b/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java
deleted file mode 100644
index 1e7d3c3..0000000
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestFileUtils.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.util;
-
-import com.google.protobuf.Message;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.apache.tajo.util.TestProtos.TestMessageProto;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestFileUtils {
- private static final String TEST_PATH = "target/test-data/TestFileUTils";
- TestMessageProto proto = null;
-
- @Before
- public void setUp() throws Exception {
- TestMessageProto.Builder builder = TestMessageProto.newBuilder();
- builder.setName("TestFileUtils");
- builder.setAge(30);
- builder.setAddr(TestFileUtils.class.getName());
-
- proto = builder.build();
-
- File testDir = new File(TEST_PATH);
- if(testDir.exists()) {
- testDir.delete();
- }
- testDir.mkdirs();
- }
-
- @After
- public void tearDown() throws Exception {
- File testDir = new File(TEST_PATH);
- if(testDir.exists()) {
- testDir.delete();
- }
- }
-
- @Test
- public final void testWriteLoadProtoFromFile() throws IOException {
- File file = new File(TEST_PATH+"/file.bin");
- file.createNewFile();
- FileUtil.writeProto(file, proto);
-
- Message defaultInstance = TestMessageProto.getDefaultInstance();
- TestMessageProto message = (TestMessageProto)
- FileUtil.loadProto(new File(TEST_PATH+"/file.bin"), defaultInstance);
-
- assertEquals(proto, message);
- }
-
- @Test
- public final void testWriteLoadProtoFromStream() throws IOException {
- FileOutputStream out = new FileOutputStream(new File(TEST_PATH+"/file.bin"));
- FileUtil.writeProto(out, proto);
-
-
- FileInputStream in = new FileInputStream(new File(TEST_PATH+"/file.bin"));
- Message defaultInstance = TestMessageProto.getDefaultInstance();
- TestMessageProto message = (TestMessageProto)
- FileUtil.loadProto(in, defaultInstance);
-
- assertEquals(proto, message);
- }
-
- @Test
- public final void testWriteLoadProtoFromPath() throws IOException {
- Path path = new Path(TEST_PATH+"/file.bin");
- Configuration conf = new Configuration();
- FileSystem localFS = FileSystem.getLocal(conf);
- FileUtil.writeProto(localFS, path, proto);
-
- Message defaultInstance = TestMessageProto.getDefaultInstance();
- TestMessageProto message = (TestMessageProto)
- FileUtil.loadProto(localFS, new Path(TEST_PATH+"/file.bin"),
- defaultInstance);
-
- assertEquals(proto, message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java
new file mode 100644
index 0000000..cefc86f
--- /dev/null
+++ b/tajo-common/src/test/java/org/apache/tajo/util/TestUriUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestUriUtil {
+ static final String URI0 = "http://192.168.0.1/table1";
+ static final String URI1 = "hbase:zk://192.168.0.1/table1";
+ static final String URI2 = "jdbc:postgresql://192.168.0.1/table1";
+
+ @Test
+ public void testGetScheme() throws Exception {
+ assertEquals("http", UriUtil.getScheme(URI0));
+ assertEquals("hbase:zk", UriUtil.getScheme(URI1));
+ assertEquals("jdbc:postgresql", UriUtil.getScheme(URI2));
+ }
+
+ @Test
+ public void testAddParam() throws Exception {
+ String userAdded = UriUtil.addParam(URI2, "user", "xxx");
+ assertEquals("jdbc:postgresql://192.168.0.1/table1?user=xxx", userAdded);
+ assertEquals("jdbc:postgresql://192.168.0.1/table1?user=xxx&pass=yyy", UriUtil.addParam(userAdded, "pass", "yyy"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
index 06a54c4..bb8cf1b 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
@@ -25,7 +25,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.storage.StorageConstants;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -61,7 +61,7 @@ public class TestDDLBuilder {
TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod1);
desc.setExternal(true);
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"),
DDLBuilder.buildDDLForExternalTable(desc));
}
@@ -87,20 +87,20 @@ public class TestDDLBuilder {
TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod2);
desc.setExternal(true);
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"),
DDLBuilder.buildDDLForExternalTable(desc));
desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod2);
desc.setExternal(false);
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"),
DDLBuilder.buildDDLForBaseTable(desc));
}
@Test
public void testBuildDDLForBaseTable() throws Exception {
TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1").toUri());
- assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"),
+ assertEquals(JavaResourceUtil.readTextFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"),
DDLBuilder.buildDDLForBaseTable(desc));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
index 8642331..760fa62 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
@@ -87,7 +87,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
}
tableSpaceUri = "hbase:zk://" + hostName + ":" + zkPort;
- HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri));
+ HBaseTablespace hBaseTablespace = new HBaseTablespace("cluster1", URI.create(tableSpaceUri), null);
hBaseTablespace.init(new TajoConf(testingCluster.getHBaseUtil().getConf()));
TablespaceManager.addTableSpaceForTest(hBaseTablespace);
}
@@ -484,7 +484,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
new ConstEval(new TextDatum("021")));
scanNode.setQual(evalNodeEq);
Tablespace tablespace = TablespaceManager.getByName("cluster1").get();
- List<Fragment> fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ List<Fragment> fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(1, fragments.size());
assertEquals("021", new String(((HBaseFragment)fragments.get(0)).getStartRow()));
assertEquals("021" + postFix, new String(((HBaseFragment)fragments.get(0)).getStopRow()));
@@ -497,7 +497,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2);
scanNode.setQual(evalNodeA);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(2, fragments.size());
HBaseFragment fragment1 = (HBaseFragment) fragments.get(0);
assertEquals("020", new String(fragment1.getStartRow()));
@@ -512,7 +512,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
new ConstEval(new TextDatum("075")));
EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3);
scanNode.setQual(evalNodeB);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(3, fragments.size());
fragment1 = (HBaseFragment) fragments.get(0);
assertEquals("020", new String(fragment1.getStartRow()));
@@ -535,7 +535,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5);
EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC);
scanNode.setQual(evalNodeD);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(3, fragments.size());
fragment1 = (HBaseFragment) fragments.get(0);
@@ -558,7 +558,7 @@ public class TestHBaseTable extends QueryTestCaseBase {
evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5);
evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC);
scanNode.setQual(evalNodeD);
- fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode);
+ fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, scanNode.getQual());
assertEquals(2, fragments.size());
fragment1 = (HBaseFragment) fragments.get(0);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
index 4dcf562..90a004b 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
@@ -37,7 +37,7 @@ import org.apache.tajo.datum.Int4Datum;
import org.apache.tajo.datum.TextDatum;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.storage.*;
-import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.KeyValueSet;
import org.junit.runners.Parameterized.Parameters;
@@ -248,7 +248,7 @@ public class TestJoinQuery extends QueryTestCaseBase {
TableMeta tableMeta = table.getMeta();
Schema schema = table.getLogicalSchema();
- String[] rows = FileUtil.readTextFileFromResource("tpch/" + tableName + ".tbl").split("\n");
+ String[] rows = JavaResourceUtil.readTextFromResource("tpch/" + tableName + ".tbl").split("\n");
assertTrue(rows.length > 0);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
index f3c302a..3a577e8 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java
@@ -33,6 +33,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.SQLSyntaxError;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.JavaResourceUtil;
import org.apache.tajo.util.Pair;
import org.junit.Rule;
import org.junit.Test;
@@ -143,7 +144,8 @@ public class TestSQLAnalyzer {
String expectedResult = "";
try {
- expectedResult = FileUtil.readTextFileFromResource("results/TestSQLAnalyzer/errors/" + fileName + ".result");
+ expectedResult =
+ JavaResourceUtil.readTextFromResource("results/TestSQLAnalyzer/errors/" + fileName + ".result");
} catch (FileNotFoundException fnfe) {
}
@@ -182,7 +184,7 @@ public class TestSQLAnalyzer {
String fileName = null;
try {
fileName = pair.getFirst().split("\\.")[0];
- expectedResult = FileUtil.readTextFileFromResource("results/TestSQLAnalyzer/" + fileName + ".result");
+ expectedResult = JavaResourceUtil.readTextFromResource("results/TestSQLAnalyzer/" + fileName + ".result");
} catch (FileNotFoundException ioe) {
expectedResult = "";
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
index e45dd75..c1d5a97 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/storage/TestRowFile.java
@@ -74,13 +74,10 @@ public class TestRowFile {
FileTablespace sm = (FileTablespace) TablespaceManager.get(cluster.getDefaultFileSystem().getUri()).get();
Path tablePath = new Path("/test");
- Path metaPath = new Path(tablePath, ".meta");
Path dataPath = new Path(tablePath, "test.tbl");
FileSystem fs = sm.getFileSystem();
fs.mkdirs(tablePath);
- FileUtil.writeProto(fs, metaPath, meta.getProto());
-
Appender appender = sm.getAppender(meta, schema, dataPath);
appender.enableStats();
appender.init();
@@ -105,14 +102,11 @@ public class TestRowFile {
assertEquals(tupleNum, stat.getNumRows().longValue());
FileStatus file = fs.getFileStatus(dataPath);
- TableProto proto = (TableProto) FileUtil.loadProto(
- cluster.getDefaultFileSystem(), metaPath, TableProto.getDefaultInstance());
- meta = new TableMeta(proto);
FileFragment fragment = new FileFragment("test.tbl", dataPath, 0, file.getLen());
int tupleCnt = 0;
start = System.currentTimeMillis();
- Scanner scanner = sm.getScanner(meta, schema, fragment);
+ Scanner scanner = sm.getScanner(meta, schema, fragment, null);
scanner.init();
while ((tuple=scanner.next()) != null) {
tupleCnt++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
index cc740e7..8b87ea9 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java
@@ -130,7 +130,7 @@ public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext>
}
}
- public EvalNode visitUnaryEval(EvalCodeGenContext context, Stack<EvalNode> stack, UnaryEval unary) {
+ public EvalNode visitUnaryEval(EvalCodeGenContext context, UnaryEval unary, Stack<EvalNode> stack) {
stack.push(unary);
if (unary.getType() == EvalType.CAST) {
visitCast(context, stack, (CastEval) unary);
@@ -349,7 +349,7 @@ public class EvalCodeGenerator extends SimpleEvalNodeVisitor<EvalCodeGenContext>
return cast;
}
- public EvalNode visitField(EvalCodeGenContext context, Stack<EvalNode> stack, FieldEval field) {
+ public EvalNode visitField(EvalCodeGenContext context, FieldEval field, Stack<EvalNode> stack) {
if (field.getValueType().getType() == TajoDataTypes.Type.NULL_TYPE) {
context.pushNullOfThreeValuedLogic();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
index 63b5445..c2a6e99 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
@@ -174,9 +174,6 @@ public class SeqScanExec extends ScanExec {
scanIt = new FilterScanIterator(scanner, qual);
} else {
- if (scanner.isSelectable()) { // TODO - isSelectable should be moved to FormatProperty
- scanner.setFilter(qual);
- }
scanIt = new FullScanIterator(scanner);
}
}
@@ -275,10 +272,17 @@ public class SeqScanExec extends ScanExec {
this.scanner = tablespace.getScanner(
meta,
plan.getPhysicalSchema(),
- fragments[0],
+ FragmentConvertor.convert(context.getConf(), fragments[0]),
projected);
}
+ if (scanner.isSelectable()) { // TODO - isSelectable should be moved to FormatProperty
+ scanner.setFilter(qual);
+ }
+
+ if (plan.hasLimit()) {
+ scanner.setLimit(plan.getLimit());
+ }
scanner.init();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index ed7698f..326bc86 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -19,6 +19,7 @@
package org.apache.tajo.master;
import com.codahale.metrics.Gauge;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -58,6 +59,7 @@ import org.apache.tajo.rule.SelfDiagnosisRuleSession;
import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.service.ServiceTrackerFactory;
import org.apache.tajo.session.SessionManager;
+import org.apache.tajo.storage.TablespaceManager;
import org.apache.tajo.util.*;
import org.apache.tajo.util.history.HistoryReader;
import org.apache.tajo.util.history.HistoryWriter;
@@ -180,11 +182,11 @@ public class TajoMaster extends CompositeService {
this.dispatcher = new AsyncDispatcher();
addIfService(dispatcher);
- // check the system directory and create if they are not created.
- checkAndInitializeSystemDirectories();
- diagnoseTajoMaster();
+ // check the system directory and create if they are not created.
+ checkAndInitializeSystemDirectories();
+ diagnoseTajoMaster();
- catalogServer = new CatalogServer(Collections.EMPTY_SET, loadFunctions());
+ catalogServer = new CatalogServer(TablespaceManager.getMetadataProviders(), loadFunctions());
addIfService(catalogServer);
catalog = new LocalCatalogWrapper(catalogServer, systemConf);
@@ -409,6 +411,14 @@ public class TajoMaster extends CompositeService {
LOG.info("Tajo Master main thread exiting");
}
+ /**
+ * This is only for unit tests.
+ */
+ @VisibleForTesting
+ public void refresh() {
+ catalogServer.refresh(TablespaceManager.getMetadataProviders());
+ }
+
public EventHandler getEventHandler() {
return dispatcher.getEventHandler();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
index 809b81f..ac40c4f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
@@ -109,7 +109,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc
FileTablespace fileTablespace = TUtil.checkTypeAndGet(tablespace, FileTablespace.class);
fragments.addAll(Repartitioner.getFragmentsFromPartitionedTable(fileTablespace, scanNode, tableDesc));
} else {
- fragments.addAll(tablespace.getSplits(tableDesc.getName(), tableDesc, scanNode));
+ fragments.addAll(tablespace.getSplits(tableDesc.getName(), tableDesc, scanNode.getQual()));
}
if (!fragments.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
index f7aff30..ffdb3f0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
@@ -291,6 +291,7 @@ public class QueryExecutor {
if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
maxRow = (int) limitNode.getFetchFirstNum();
+ scanNode.setLimit(maxRow);
}
if (desc.getStats().getNumRows() == 0) {
desc.getStats().setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
index c4fc645..3ce86b1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
@@ -19,6 +19,7 @@
package org.apache.tajo.querymaster;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -113,7 +114,7 @@ public class Repartitioner {
// So, we need to handle FileFragment by its size.
// If we don't check its size, it can cause IndexOutOfBoundsException.
Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
- List<Fragment> fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc);
+ List<Fragment> fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc, null);
if (fileFragments.size() > 0) {
fragments[i] = fileFragments.get(0);
} else {
@@ -389,8 +390,8 @@ public class Repartitioner {
} else {
- Collection<Fragment> scanFragments = space.getSplits(eachScan.getCanonicalName(),
- tableDesc, eachScan);
+ Collection<Fragment> scanFragments =
+ space.getSplits(eachScan.getCanonicalName(), tableDesc, eachScan.getQual());
if (scanFragments != null) {
rightFragments.addAll(scanFragments);
}
@@ -460,22 +461,24 @@ public class Repartitioner {
/**
* It creates a number of fragments for all partitions.
*/
- public static List<Fragment> getFragmentsFromPartitionedTable(FileTablespace sm,
+ public static List<Fragment> getFragmentsFromPartitionedTable(Tablespace tsHandler,
ScanNode scan,
TableDesc table) throws IOException {
+ Preconditions.checkArgument(tsHandler instanceof FileTablespace, "tsHandler must be FileTablespace");
if (!(scan instanceof PartitionedTableScanNode)) {
throw new IllegalArgumentException("scan should be a PartitionedTableScanNode type.");
}
List<Fragment> fragments = Lists.newArrayList();
PartitionedTableScanNode partitionsScan = (PartitionedTableScanNode) scan;
- fragments.addAll(sm.getSplits(
+ fragments.addAll(((FileTablespace) tsHandler).getSplits(
scan.getCanonicalName(), table.getMeta(), table.getSchema(), partitionsScan.getInputPaths()));
partitionsScan.setInputPaths(null);
return fragments;
}
private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext schedulerContext, Stage stage,
- int baseScanId, Fragment[] fragments) throws IOException {
+ int baseScanId, Fragment[] fragments)
+ throws IOException, TajoException {
ExecutionBlock execBlock = stage.getBlock();
ScanNode[] scans = execBlock.getScanNodes();
@@ -499,20 +502,20 @@ public class Repartitioner {
for (int i = 0; i < scans.length; i++) {
ScanNode scan = scans[i];
TableDesc desc = stage.getContext().getTableDesc(scan);
- TableMeta meta = desc.getMeta();
Collection<Fragment> scanFragments;
Path[] partitionScanPaths = null;
- FileTablespace space = (FileTablespace) TablespaceManager.get(desc.getUri()).get();
+
+ Tablespace space = TablespaceManager.get(desc.getUri()).get();
if (scan.getType() == NodeType.PARTITIONS_SCAN) {
- PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)scan;
+ PartitionedTableScanNode partitionScan = (PartitionedTableScanNode) scan;
partitionScanPaths = partitionScan.getInputPaths();
// set null to inputPaths in getFragmentsFromPartitionedTable()
scanFragments = getFragmentsFromPartitionedTable(space, scan, desc);
} else {
- scanFragments = space.getSplits(scan.getCanonicalName(), desc, scan);
+ scanFragments = space.getSplits(scan.getCanonicalName(), desc, scan.getQual());
}
if (scanFragments != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 68916e4..125c83c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -1144,7 +1144,7 @@ public class Stage implements EventHandler<StageEvent> {
// After calling this method, partition paths are removed from the physical plan.
fragments = Repartitioner.getFragmentsFromPartitionedTable((FileTablespace) tablespace, scan, table);
} else {
- fragments = tablespace.getSplits(scan.getCanonicalName(), table, scan);
+ fragments = tablespace.getSplits(scan.getCanonicalName(), table, scan.getQual());
}
Stage.scheduleFragments(stage, fragments);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml
index fdce467..b0e9f64 100644
--- a/tajo-dist/pom.xml
+++ b/tajo-dist/pom.xml
@@ -143,7 +143,8 @@
run cp -r $ROOT/tajo-catalog/target/tajo-catalog-${project.version}/* .
run cp -r $ROOT/tajo-storage/target/tajo-storage-${project.version}/* .
run cp -r $ROOT/tajo-sql-parser/target/tajo-sql-parser-${project.version}/* .
-
+ run cp -r $ROOT/tajo-storage/tajo-storage-jdbc/target/tajo-storage-jdbc-${project.version}.jar .
+ run cp -r $ROOT/tajo-storage/tajo-storage-pgsql/target/tajo-storage-pgsql-${project.version}.jar .
run cp -r $ROOT/tajo-pullserver/target/tajo-pullserver-${project.version}.jar .
run cp -r $ROOT/tajo-metrics/target/tajo-metrics-${project.version}.jar .
run cp -r $ROOT/tajo-core/target/tajo-core-${project.version}.jar .
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/index.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst
index ec65fd3..5a54809 100644
--- a/tajo-docs/src/main/sphinx/index.rst
+++ b/tajo-docs/src/main/sphinx/index.rst
@@ -37,7 +37,7 @@ Table of Contents:
functions
table_management
table_partitioning
- storage_plugin
+ storage_plugins
index_overview
backup_and_restore
hive_integration
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugin.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugin.rst b/tajo-docs/src/main/sphinx/storage_plugin.rst
deleted file mode 100644
index d9c6838..0000000
--- a/tajo-docs/src/main/sphinx/storage_plugin.rst
+++ /dev/null
@@ -1,47 +0,0 @@
-*************************************
-Storage Plugin
-*************************************
-
-Overview
-========
-
-Tajo supports various storage systems, such as HDFS, Amazon S3, Openstack Swift, and HBase. Also, we have a plan to support RDBMS storages like Oracle, MySQL, PostgreSQL. Tajo already embeds HDFS, S3, Openstack, and HBase, and also Tajo allows users to register custom storages and data formats to Tajo cluster instances. This section describes how you register custom storages and data types.
-
-Register custom storage
-=======================
-
-First of all, your storage implementation should be packed as a jar file. Then, please copy the jar file into ``tajo/extlib`` directory. Next, you should copy ``conf/storage-site.json.template`` into ``conf/storage-site.json`` and modify the file like the below.
-
-Configuration
-=============
-
-Tajo has a default configuration for builtin storages, such as HDFS, local file system, and Amazon S3. it also allows users to add custom storage plugins
-
-``conf/storage-site.json`` file has the following struct:
-
-.. code-block:: json
-
- {
- "storages": {
- "${scheme}": {
- "handler": "${class name}"
- }
- }
- }
-
-Each storage instance (i.e., :doc:`/table_management/tablespaces`) is identified by an URI. The scheme of URI plays a role to identify storage type. For example, ``hdfs://`` is used for Hdfs storage, ``jdbc://`` is used for JDBC-based storage, and ``hbase://`` is used for HBase storage.
-
-You should substitute a scheme name without ``://`` for ``${scheme}``.
-
-See an example for HBase storage.
-
-.. code-block:: json
-
- {
- "storages": {
- "hbase": {
- "handler": "org.apache.tajo.storage.hbase.HBaseTablespace",
- "default-format": "hbase"
- }
- }
- }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugins.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugins.rst b/tajo-docs/src/main/sphinx/storage_plugins.rst
new file mode 100644
index 0000000..35b3d38
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/storage_plugins.rst
@@ -0,0 +1,11 @@
+*************************************
+Storage Plugin
+*************************************
+
+This section describes the storage plugins available in Tajo to access datasets from different data sources.
+
+.. toctree::
+ :maxdepth: 1
+
+ storage_plugins/overview
+ storage_plugins/postgresql
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugins/overview.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugins/overview.rst b/tajo-docs/src/main/sphinx/storage_plugins/overview.rst
new file mode 100644
index 0000000..243d144
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/storage_plugins/overview.rst
@@ -0,0 +1,47 @@
+*************************************
+Storage Plugin Overview
+*************************************
+
+Overview
+========
+
+Tajo supports various storage systems, such as HDFS, Amazon S3, Openstack Swift, HBase, and RDBMS. Tajo already embeds HDFS, S3, Openstack, HBase, RDBMS storage plugins, and also Tajo allows users to register custom storages and data formats to Tajo cluster instances. This section describes how you register custom storages and data types.
+
+Register custom storage
+=======================
+
+First of all, your storage implementation should be packed as a jar file. Then, please copy the jar file into ``tajo/extlib`` directory. Next, you should copy ``conf/storage-site.json.template`` into ``conf/storage-site.json`` and modify the file like the below.
+
+Configuration
+=============
+
+Tajo has a default configuration for builtin storages, such as HDFS, local file system, and Amazon S3. it also allows users to add custom storage plugins
+
+``conf/storage-site.json`` file has the following struct:
+
+.. code-block:: json
+
+ {
+ "storages": {
+ "${scheme}": {
+ "handler": "${class name}"
+ }
+ }
+ }
+
+Each storage instance (i.e., :doc:`/table_management/tablespaces`) is identified by an URI. The scheme of URI plays a role to identify storage type. For example, ``hdfs://`` is used for Hdfs storage, ``jdbc://`` is used for JDBC-based storage, and ``hbase://`` is used for HBase storage.
+
+You should substitute a scheme name without ``://`` for ``${scheme}``.
+
+See an example for HBase storage.
+
+.. code-block:: json
+
+ {
+ "storages": {
+ "hbase": {
+ "handler": "org.apache.tajo.storage.hbase.HBaseTablespace",
+ "default-format": "hbase"
+ }
+ }
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0e4ad563/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst b/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst
new file mode 100644
index 0000000..9c9a22b
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/storage_plugins/postgresql.rst
@@ -0,0 +1,40 @@
+*************************************
+PostgreSQL Storage Handler
+*************************************
+
+Overview
+========
+
+PostgreSQL storage handler is available by default in Tajo. It enables users' queries to access database objects in PostgreSQL. Tables in PostgreSQL will be shown as tables in Tajo too. Most of the SQL queries used for PostgreSQL are available in Tajo via this storage handles. Its main advantages is to allow federated query processing among tables in stored HDFS and PostgreSQL.
+
+Configuration
+=============
+
+PostgreSQL storage handler is a builtin storage handler. So, you can eaisly register PostgreSQL databases to a Tajo cluster if you just add the following line to ``conf/storage-site.json`` file. If you want to know more information about ``storage-site.json``, please refer to :doc:`/table_management/tablespaces`.
+
+.. code-block:: json
+
+ {
+ "spaces": {
+ "pgsql_db1": {
+ "uri": "jdbc:postgresql://hostname:port/db1"
+
+ "configs": {
+ "mapped_database": "tajo_db1"
+ "connection_properties": {
+ "user": "tajo",
+ "password": "xxxx"
+ }
+ }
+ }
+ }
+ }
+
+``configs`` allows users to specific additional configurations.
+``mapped_database`` specifies a database name shown in Tajo. In the example, the database ``db1`` in PostgreSQL
+will be mapped to the database ``tajo_db1`` in Tajo.
+``connection_properties`` allows users to set JDBC connection parameters.
+Please refer to https://jdbc.postgresql.org/documentation/head/connect.html in order to know the details of
+PostgreSQL connection parameters.
+
+The storage-site.json will be effective after you restart a tajo cluster.
\ No newline at end of file