You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2014/04/15 11:59:18 UTC
[2/2] git commit: TAJO-753: Clean up of maven dependencies. (jinho)
TAJO-753: Clean up of maven dependencies. (jinho)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/646cf1ad
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/646cf1ad
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/646cf1ad
Branch: refs/heads/branch-0.8.0
Commit: 646cf1ade76379f62d34657d032d6ee1ef8088f0
Parents: 031d5dd
Author: jinossy <ji...@gmail.com>
Authored: Tue Apr 15 18:58:00 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Tue Apr 15 18:58:00 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
tajo-algebra/pom.xml | 17 +-
tajo-catalog/tajo-catalog-client/pom.xml | 19 +-
tajo-catalog/tajo-catalog-common/pom.xml | 31 +--
.../apache/tajo/catalog/CatalogConstants.java | 49 ----
.../org/apache/tajo/catalog/CatalogUtil.java | 26 ---
tajo-catalog/tajo-catalog-drivers/pom.xml | 9 -
.../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 225 ++-----------------
.../tajo/catalog/store/HCatalogStore.java | 51 +++--
.../tajo/catalog/store/TestHCatalogStore.java | 37 +--
tajo-catalog/tajo-catalog-server/pom.xml | 26 ++-
tajo-client/pom.xml | 80 +------
tajo-common/pom.xml | 23 +-
.../org/apache/tajo/datum/TestInt2Datum.java | 6 +-
tajo-core/pom.xml | 1 +
tajo-core/tajo-core-backend/pom.xml | 77 ++-----
.../java/org/apache/tajo/benchmark/TPCH.java | 4 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 4 +-
.../tajo/engine/planner/LogicalPlanner.java | 5 +-
.../test/java/org/apache/tajo/TpchTestBase.java | 4 +-
.../org/apache/tajo/client/TestDDLBuilder.java | 9 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 8 +-
.../tajo/engine/query/TestNullValues.java | 12 +-
tajo-core/tajo-core-pullserver/pom.xml | 20 +-
.../pullserver/FadvisedFileRegionWrapper.java | 35 +++
.../pullserver/listener/FileCloseListener.java | 1 +
.../pullserver/FadvisedFileRegionWrapper.java | 34 +++
.../pullserver/FadvisedFileRegionWrapper.java | 36 +++
.../pullserver/listener/FileCloseListener.java | 42 ++++
.../tajo/pullserver/PullServerAuxService.java | 2 +-
.../tajo/pullserver/TajoPullServerService.java | 3 +-
tajo-jdbc/pom.xml | 30 +--
tajo-project/pom.xml | 39 +++-
tajo-rpc/pom.xml | 38 +---
.../org/apache/tajo/rpc/ServerCallable.java | 5 +-
tajo-storage/pom.xml | 64 +++---
.../tajo/storage/AbstractStorageManager.java | 13 +-
.../java/org/apache/tajo/storage/CSVFile.java | 16 +-
.../apache/tajo/storage/StorageConstants.java | 70 ++++++
.../org/apache/tajo/storage/StorageUtil.java | 28 ++-
.../org/apache/tajo/storage/rcfile/RCFile.java | 17 +-
.../sequencefile/SequenceFileAppender.java | 12 +-
.../sequencefile/SequenceFileScanner.java | 7 +-
.../apache/tajo/storage/TestMergeScanner.java | 2 +-
.../apache/tajo/storage/TestStorageManager.java | 4 +-
.../org/apache/tajo/storage/TestStorages.java | 22 +-
.../apache/tajo/storage/v2/TestStorages.java | 6 +-
47 files changed, 540 insertions(+), 731 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9f4dd22..6e1a178 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -629,6 +629,8 @@ Release 0.8.0 - unreleased
TASKS
+ TAJO-753: Clean up of maven dependencies. (jinho)
+
TAJO-529: Fix warnings in tajo-algebra. (jaehwa)
TAJO-700: Update site, wikis, pom.xml and other resources to point to the
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-algebra/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-algebra/pom.xml b/tajo-algebra/pom.xml
index 2f91b85..dea4727 100644
--- a/tajo-algebra/pom.xml
+++ b/tajo-algebra/pom.xml
@@ -68,12 +68,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <excludeGroupIds>org.apache.hadoop</excludeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -98,24 +97,10 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-client/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/pom.xml b/tajo-catalog/tajo-catalog-client/pom.xml
index b511739..66664d4 100644
--- a/tajo-catalog/tajo-catalog-client/pom.xml
+++ b/tajo-catalog/tajo-catalog-client/pom.xml
@@ -128,23 +128,14 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-rpc</artifactId>
</dependency>
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-common/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/pom.xml b/tajo-catalog/tajo-catalog-common/pom.xml
index fa90237..233079d 100644
--- a/tajo-catalog/tajo-catalog-common/pom.xml
+++ b/tajo-catalog/tajo-catalog-common/pom.xml
@@ -117,6 +117,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
@@ -139,10 +140,10 @@
<artifactId>tajo-common</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-rpc</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
-
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -152,30 +153,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>parquet-column</artifactId>
- <version>${parquet.version}</version>
- </dependency>
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>parquet-hadoop</artifactId>
- <version>${parquet.version}</version>
- </dependency>
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>parquet-format</artifactId>
- <version>${parquet.format.version}</version>
- </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
index 8cad4fe..d823f25 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
@@ -18,9 +18,6 @@
package org.apache.tajo.catalog;
-import parquet.hadoop.ParquetWriter;
-import parquet.hadoop.metadata.CompressionCodecName;
-
public class CatalogConstants {
public final static String IDENTIFIER_DELIMITER_REGEXP = "\\.";
public final static String IDENTIFIER_DELIMITER = ".";
@@ -54,50 +51,4 @@ public class CatalogConstants {
public static final String COL_DATABASES_PK = "DB_ID";
public static final String COL_TABLES_PK = "TID";
public static final String COL_TABLES_NAME = "TABLE_NAME";
-
- // table options
- public static final String COMPRESSION_CODEC = "compression.codec";
- public static final String COMPRESSION_TYPE = "compression.type";
-
- public static final String CSVFILE_DELIMITER = "csvfile.delimiter";
- public static final String CSVFILE_NULL = "csvfile.null";
- public static final String CSVFILE_SERDE = "csvfile.serde";
-
-
- public static final String SEQUENCEFILE_DELIMITER = "sequencefile.delimiter";
- public static final String SEQUENCEFILE_NULL = "sequencefile.null";
- public static final String SEQUENCEFILE_SERDE = "sequencefile.serde";
-
- public static final String RCFILE_NULL = "rcfile.null";
- public static final String RCFILE_SERDE = "rcfile.serde";
-
- public static final String DEFAULT_FIELD_DELIMITER = "|";
- public static final String DEFAULT_BINARY_SERDE = "org.apache.tajo.storage.BinarySerializerDeserializer";
- public static final String DEFAULT_TEXT_SERDE = "org.apache.tajo.storage.TextSerializerDeserializer";
-
- public static final String PARQUET_DEFAULT_BLOCK_SIZE;
- public static final String PARQUET_DEFAULT_PAGE_SIZE;
- public static final String PARQUET_DEFAULT_COMPRESSION_CODEC_NAME;
- public static final String PARQUET_DEFAULT_IS_DICTIONARY_ENABLED;
- public static final String PARQUET_DEFAULT_IS_VALIDATION_ENABLED;
-
- static {
- PARQUET_DEFAULT_BLOCK_SIZE =
- Integer.toString(ParquetWriter.DEFAULT_BLOCK_SIZE);
- PARQUET_DEFAULT_PAGE_SIZE =
- Integer.toString(ParquetWriter.DEFAULT_PAGE_SIZE);
-
- // When parquet-hadoop 1.3.3 is available, this should be changed to
- // ParquetWriter.DEFAULT_COMPRESSION_CODEC_NAME.
- PARQUET_DEFAULT_COMPRESSION_CODEC_NAME =
- CompressionCodecName.UNCOMPRESSED.name().toLowerCase();
-
- // When parquet-hadoop 1.3.3 is available, this should be changed to
- // ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED.
- PARQUET_DEFAULT_IS_DICTIONARY_ENABLED = "true";
-
- // When parquet-hadoop 1.3.3 is available, this should be changed to
- // ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED.
- PARQUET_DEFAULT_IS_VALIDATION_ENABLED = "false";
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 7555325..7568b26 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -36,8 +36,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import parquet.hadoop.ParquetOutputFormat;
-
import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import static org.apache.tajo.common.TajoDataTypes.Type;
@@ -293,30 +291,6 @@ public class CatalogUtil {
return null;
}
}
- public static Options newOptionsWithDefault(StoreType type) {
- Options options = new Options();
- if(StoreType.CSV == type){
- options.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
- } else if(StoreType.RCFILE == type){
- options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.DEFAULT_BINARY_SERDE);
- } else if(StoreType.SEQUENCEFILE == type){
- options.put(CatalogConstants.SEQUENCEFILE_SERDE, CatalogConstants.DEFAULT_TEXT_SERDE);
- options.put(CatalogConstants.SEQUENCEFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
- } else if (type == StoreType.PARQUET) {
- options.put(ParquetOutputFormat.BLOCK_SIZE,
- CatalogConstants.PARQUET_DEFAULT_BLOCK_SIZE);
- options.put(ParquetOutputFormat.PAGE_SIZE,
- CatalogConstants.PARQUET_DEFAULT_PAGE_SIZE);
- options.put(ParquetOutputFormat.COMPRESSION,
- CatalogConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME);
- options.put(ParquetOutputFormat.ENABLE_DICTIONARY,
- CatalogConstants.PARQUET_DEFAULT_IS_DICTIONARY_ENABLED);
- options.put(ParquetOutputFormat.VALIDATION,
- CatalogConstants.PARQUET_DEFAULT_IS_VALIDATION_ENABLED);
- }
-
- return options;
- }
public static TableMeta newTableMeta(StoreType type) {
return new TableMeta(type, new Options());
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/pom.xml b/tajo-catalog/tajo-catalog-drivers/pom.xml
index 157de68..97b8062 100644
--- a/tajo-catalog/tajo-catalog-drivers/pom.xml
+++ b/tajo-catalog/tajo-catalog-drivers/pom.xml
@@ -47,15 +47,6 @@
<profiles>
<profile>
- <id>hcatalog-0.11.0</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>tajo-hcatalog</module>
- </modules>
- </profile>
- <profile>
<id>hcatalog-0.12.0</id>
<activation>
<activeByDefault>false</activeByDefault>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
index 86ba343..e8f6855 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
@@ -57,6 +57,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
@@ -94,27 +95,14 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-rpc</artifactId>
</dependency>
-
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>10.8.2.2</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
@@ -128,189 +116,22 @@
<version>0.9.0</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<profiles>
<profile>
- <id>hcatalog-0.11.0</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <hive.version>0.11.0</hive.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- <version>2.3-eb</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-contrib</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-hbase-handler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-testutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libfb303</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shimss</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libfb303</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hcatalog</groupId>
- <artifactId>hcatalog-core</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
<id>hcatalog-0.12.0</id>
<activation>
<activeByDefault>false</activeByDefault>
@@ -365,6 +186,10 @@
<groupId>com.jolbox</groupId>
<artifactId>bonecp</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -433,6 +258,10 @@
<groupId>com.jolbox</groupId>
<artifactId>bonecp</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -475,12 +304,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
</dependencies>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 0fe255c..4880447 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -46,6 +46,7 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.exception.NotImplementedException;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.InternalException;
+import org.apache.tajo.storage.StorageConstants;
import java.io.IOException;
import java.util.*;
@@ -190,24 +191,24 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat));
if (storeType.equals(CatalogProtos.StoreType.CSV)) {
- options.put(CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
} else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) {
- options.put(RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.put(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) {
- options.put(RCFILE_SERDE, DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (ColumnarSerDe.class.getName().equals(serde)) {
- options.put(RCFILE_SERDE, DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
} else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) {
- options.put(SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.put(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.put(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinarySerDe.class.getName().equals(serde)) {
- options.put(SEQUENCEFILE_SERDE, DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (LazySimpleSerDe.class.getName().equals(serde)) {
- options.put(SEQUENCEFILE_SERDE, DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
}
@@ -254,7 +255,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
}
} finally {
- client.release();
+ if(client != null) client.release();
}
TableMeta meta = new TableMeta(storeType, options);
@@ -288,7 +289,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
} catch (MetaException e) {
throw new CatalogException(e);
} finally {
- client.release();
+ if(client != null) client.release();
}
}
@@ -465,26 +466,27 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) {
- String serde = tableDesc.getMeta().getOption(RCFILE_SERDE);
+ String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
- if (DEFAULT_TEXT_SERDE.equals(serde)) {
+ if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
} else {
sd.getSerdeInfo().setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(RCFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) {
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
- StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(RCFILE_NULL)));
+ StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV)) {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
- String fieldDelimiter = tableDesc.getMeta().getOption(CSVFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
@@ -496,19 +498,20 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
StringEscapeUtils.unescapeJava(fieldDelimiter));
- if (tableDesc.getMeta().getOption(CSVFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) {
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
- StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(CSVFILE_NULL)));
+ StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE)) {
- String serde = tableDesc.getMeta().getOption(SEQUENCEFILE_SERDE);
+ String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName());
- if (DEFAULT_TEXT_SERDE.equals(serde)) {
+ if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
- String fieldDelimiter = tableDesc.getMeta().getOption(SEQUENCEFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
@@ -523,9 +526,9 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(SEQUENCEFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) {
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
- StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(SEQUENCEFILE_NULL)));
+ StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
}
} else {
throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType().name()));
@@ -540,7 +543,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
} catch (Exception e) {
throw new CatalogException(e);
} finally {
- client.release();
+ if(client != null) client.release();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index f4ca4d1..0cb4520 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -29,6 +29,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -104,15 +105,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(StringEscapeUtils.escapeJava(CatalogConstants.DEFAULT_FIELD_DELIMITER),
- table1.getMeta().getOption(CatalogConstants.CSVFILE_DELIMITER));
+ assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER),
+ table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER));
store.dropTable(DB_NAME, CUSTOMER);
}
@Test
public void testTableUsingRCFileWithBinarySerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -133,15 +134,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_BINARY_SERDE,
- table1.getMeta().getOption(CatalogConstants.RCFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_BINARY_SERDE,
+ table1.getMeta().getOption(StorageConstants.RCFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
@Test
public void testTableUsingRCFileWithTextSerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -162,15 +163,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(CatalogConstants.RCFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.RCFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
@Test
public void testTableWithNullValue() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001"));
- options.put(CatalogConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N"));
+ options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001"));
+ options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N"));
TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
@@ -198,11 +199,11 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(table.getMeta().getOption(CatalogConstants.CSVFILE_DELIMITER),
- table1.getMeta().getOption(CatalogConstants.CSVFILE_DELIMITER));
+ assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER),
+ table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER));
- assertEquals(table.getMeta().getOption(CatalogConstants.CSVFILE_NULL),
- table1.getMeta().getOption(CatalogConstants.CSVFILE_NULL));
+ assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_NULL),
+ table1.getMeta().getOption(StorageConstants.CSVFILE_NULL));
store.dropTable(DB_NAME, SUPPLIER);
}
@@ -304,7 +305,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingSequenceFileWithBinarySerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.SEQUENCEFILE_SERDE, CatalogConstants.DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -325,15 +326,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_BINARY_SERDE,
- table1.getMeta().getOption(CatalogConstants.SEQUENCEFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_BINARY_SERDE,
+ table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
@Test
public void testTableUsingSequenceFileWithTextSerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.SEQUENCEFILE_SERDE, CatalogConstants.DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -354,7 +355,7 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(CatalogConstants.SEQUENCEFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-server/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml
index 03e92ec..dfbec6d 100644
--- a/tajo-catalog/tajo-catalog-server/pom.xml
+++ b/tajo-catalog/tajo-catalog-server/pom.xml
@@ -94,6 +94,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
@@ -127,7 +128,21 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-rpc</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>zookeeper</artifactId>
+ <groupId>org.apache.zookeeper</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -138,17 +153,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
- <version>10.8.2.2</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-client/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-client/pom.xml b/tajo-client/pom.xml
index c6fbb27..6a14382 100644
--- a/tajo-client/pom.xml
+++ b/tajo-client/pom.xml
@@ -157,12 +157,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <excludeGroupIds>org.apache.hadoop</excludeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -180,20 +179,6 @@
<dependency>
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- <exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tajo</groupId>
@@ -210,59 +195,28 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
@@ -273,18 +227,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
-
- <dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
- <version>2.11</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-common/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml
index 00e2448..35ac3e5 100644
--- a/tajo-common/pom.xml
+++ b/tajo-common/pom.xml
@@ -129,19 +129,15 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<dependencies>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
@@ -150,20 +146,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<artifactId>commons-logging-api</artifactId>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <type>jar</type>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
- <type>jar</type>
- <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
index 832f563..6af9700 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
@@ -18,12 +18,10 @@
package org.apache.tajo.datum;
-import org.junit.Test;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.junit.Test;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class TestInt2Datum {
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml
index e131d0e..709390d 100644
--- a/tajo-core/pom.xml
+++ b/tajo-core/pom.xml
@@ -119,6 +119,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<excludeGroupIds>org.apache.tajo</excludeGroupIds>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/pom.xml b/tajo-core/tajo-core-backend/pom.xml
index 86ea4daf..7e43bac 100644
--- a/tajo-core/tajo-core-backend/pom.xml
+++ b/tajo-core/tajo-core-backend/pom.xml
@@ -171,12 +171,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <excludeGroupIds>org.apache.hadoop</excludeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -235,6 +234,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -293,19 +293,22 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <scope>compile</scope>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -315,6 +318,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -333,75 +337,29 @@
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4</artifactId>
</dependency>
<dependency>
- <groupId>org.antlr</groupId>
- <artifactId>antlr4-runtime</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>10.8.2.2</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </dependency>
-
<dependency>
<groupId>org.dspace.dependencies</groupId>
<artifactId>dspace-geoip</artifactId>
<version>1.2.3</version>
</dependency>
-
<dependency>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
<version>3.1.1</version>
</dependency>
-
-
- <dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- <version>2.5-20081211</version>
- </dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
@@ -409,21 +367,10 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- <version>6.1.14</version>
- </dependency>
- <dependency>
- <groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
<version>6.1.14</version>
</dependency>
<dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-api-2.1</artifactId>
- <version>6.1.14</version>
- </dependency>
-
- <dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${metrics.version}</version>
@@ -660,6 +607,10 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
index 5dd0b12..abd9e7f 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
@@ -23,12 +23,12 @@ import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.storage.StorageConstants;
import java.io.IOException;
import java.sql.SQLException;
@@ -189,7 +189,7 @@ public class TPCH extends BenchmarkSet {
private void loadTable(String tableName) throws ServiceException {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
- meta.putOption(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ meta.putOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
try {
tajo.createExternalTable(tableName, getSchema(tableName), new Path(dataDir, tableName), meta);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index f28f7ca..5d75edd 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -28,9 +28,9 @@ import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tajo.algebra.*;
import org.apache.tajo.algebra.Aggregation.GroupType;
import org.apache.tajo.algebra.LiteralValue.LiteralType;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.engine.parser.SQLParser.*;
+import org.apache.tajo.storage.StorageConstants;
import java.nio.charset.Charset;
import java.util.*;
@@ -1324,7 +1324,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
public Map<String, String> escapeTableMeta(Map<String, String> map) {
Map<String, String> params = new HashMap<String, String>();
for (Map.Entry<String, String> entry : map.entrySet()) {
- if (entry.getKey().equals(CatalogConstants.CSVFILE_DELIMITER)) {
+ if (entry.getKey().equals(StorageConstants.CSVFILE_DELIMITER)) {
params.put(entry.getKey(), escapeDelimiter(entry.getValue()));
} else {
params.put(entry.getKey(), entry.getValue());
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 8946c02..f2ddf13 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -42,6 +42,7 @@ import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule;
import org.apache.tajo.engine.utils.SchemaUtil;
import org.apache.tajo.master.session.Session;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -1321,8 +1322,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
}
- // Set default options to be created.
- Options options = CatalogUtil.newOptionsWithDefault(createTableNode.getStorageType());
+ // Set default storage properties to be created.
+ Options options = StorageUtil.newPhysicalProperties(createTableNode.getStorageType());
if (expr.hasParams()) {
options.putAll(expr.getParams());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
index 8487e84..dd21385 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
@@ -22,9 +22,9 @@ import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.benchmark.TPCH;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.FileUtil;
import java.io.File;
@@ -91,7 +91,7 @@ public class TpchTestBase {
private void setUp() throws Exception {
util = new LocalTajoTestingUtility();
Options opt = new Options();
- opt.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ opt.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
util.setup(names, paths, schemas, opt);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
index 9d8d631..1855217 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
@@ -24,12 +24,11 @@ import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.FileUtil;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class TestDDLBuilder {
@@ -43,8 +42,8 @@ public class TestDDLBuilder {
schema1.addColumn("addr", TajoDataTypes.Type.TEXT);
meta1 = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
- meta1.putOption(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
- meta1.putOption(CatalogConstants.COMPRESSION_CODEC, GzipCodec.class.getName());
+ meta1.putOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName());
Schema expressionSchema = new Schema();
expressionSchema.addColumn("key", TajoDataTypes.Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index 4dc5728..a4e31e0 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -26,9 +26,13 @@ import org.apache.tajo.IntegrationTest;
import org.apache.tajo.QueryTestCaseBase;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.CatalogService;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Options;
+import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.storage.StorageConstants;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -207,7 +211,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
Options options = desc.getMeta().getOptions();
assertNotNull(options);
- assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(CatalogConstants.CSVFILE_DELIMITER));
+ assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(StorageConstants.CSVFILE_DELIMITER));
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
index c460bcf..c5afc3d 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
@@ -20,10 +20,10 @@ package org.apache.tajo.engine.query;
import org.apache.tajo.IntegrationTest;
import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.storage.StorageConstants;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -52,7 +52,7 @@ public class TestNullValues {
"3|filled|0.2"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ opts.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable1 where col3 is null");
@@ -78,7 +78,7 @@ public class TestNullValues {
"3|filled|"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ opts.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable2 where col1 is not null");
@@ -113,7 +113,7 @@ public class TestNullValues {
",,,43578,19,13,6,3581,2557,1024"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, ",");
+ opts.put(StorageConstants.CSVFILE_DELIMITER, ",");
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable3 where col1 is null and col2 is null and col3 is null and col4 = 43578");
@@ -146,8 +146,8 @@ public class TestNullValues {
",\\N,,43578"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, ",");
- opts.put(CatalogConstants.CSVFILE_NULL, "\\\\N");
+ opts.put(StorageConstants.CSVFILE_DELIMITER, ",");
+ opts.put(StorageConstants.CSVFILE_NULL, "\\\\N");
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable4 where col1 is null and col2 is null and col3 is null and col5 is null and col4 = 43578");
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/pom.xml b/tajo-core/tajo-core-pullserver/pom.xml
index b5049be..cdc5ff6 100644
--- a/tajo-core/tajo-core-pullserver/pom.xml
+++ b/tajo-core/tajo-core-pullserver/pom.xml
@@ -34,6 +34,10 @@
<dependencies>
<dependency>
<groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-rpc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
<artifactId>tajo-catalog-common</artifactId>
<scope>provided</scope>
</dependency>
@@ -42,7 +46,6 @@
<artifactId>tajo-storage</artifactId>
<scope>provided</scope>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
@@ -59,21 +62,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
new file mode 100644
index 0000000..335dff0
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver;
+
+import org.apache.hadoop.io.ReadaheadPool;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+@Deprecated
+public class FadvisedFileRegionWrapper extends org.apache.hadoop.mapred.FadvisedFileRegion {
+
+
+ public FadvisedFileRegionWrapper(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier) throws IOException {
+ super(file, position, count, manageOsCache, readaheadLength, readaheadPool, identifier);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
index b7f3e34..0933167 100644
--- a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.mapred.FadvisedFileRegion;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
+@Deprecated
public class FileCloseListener implements ChannelFutureListener {
private FadvisedFileRegion filePart;
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
new file mode 100644
index 0000000..42d1cd8
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver;
+
+import org.apache.hadoop.io.ReadaheadPool;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+public class FadvisedFileRegionWrapper extends org.apache.hadoop.mapred.FadvisedFileRegion {
+
+
+ public FadvisedFileRegionWrapper(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier) throws IOException {
+ super(file, position, count, manageOsCache, readaheadLength, readaheadPool, identifier);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
new file mode 100644
index 0000000..ee53bc6
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver;
+
+import org.apache.hadoop.io.ReadaheadPool;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/* tajo is not yet supported on Windows */
+public class FadvisedFileRegionWrapper extends org.apache.hadoop.mapred.FadvisedFileRegion {
+ public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;
+
+ public FadvisedFileRegionWrapper(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier) throws IOException {
+ super(file, position, count, manageOsCache, readaheadLength, readaheadPool,
+ identifier, DEFAULT_SHUFFLE_BUFFER_SIZE, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
new file mode 100644
index 0000000..5b2d1b3
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver.listener;
+
+import org.apache.hadoop.mapred.FadvisedFileRegion;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+
+public class FileCloseListener implements ChannelFutureListener {
+
+ private FadvisedFileRegion filePart;
+
+ public FileCloseListener(FadvisedFileRegion filePart) {
+ this.filePart = filePart;
+ }
+
+ // TODO error handling; distinguish IO/connection failures,
+ // attribute to appropriate spill output
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if(future.isSuccess()){
+ filePart.transferSuccessful();
+ }
+ filePart.releaseExternalResources();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
index 40978fe..87011da 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
@@ -490,7 +490,7 @@ public class PullServerAuxService extends AuxiliaryService {
}
ChannelFuture writeFuture;
if (ch.getPipeline().get(SslHandler.class) == null) {
- final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
+ final FadvisedFileRegionWrapper partition = new FadvisedFileRegionWrapper(spill,
file.startOffset, file.length(), manageOsCache, readaheadLength,
readaheadPool, file.getFile().getAbsolutePath());
writeFuture = ch.write(partition);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
index 70f5421..cc3cb2e 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.mapred.FadvisedChunkedFile;
-import org.apache.hadoop.mapred.FadvisedFileRegion;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -487,7 +486,7 @@ public class TajoPullServerService extends AbstractService {
}
ChannelFuture writeFuture;
if (ch.getPipeline().get(SslHandler.class) == null) {
- final FadvisedFileRegion filePart = new FadvisedFileRegion(spill,
+ final FadvisedFileRegionWrapper filePart = new FadvisedFileRegionWrapper(spill,
file.startOffset, file.length(), manageOsCache, readaheadLength,
readaheadPool, file.getFile().getAbsolutePath());
writeFuture = ch.write(filePart);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml
index d691af7..7a615a0 100644
--- a/tajo-jdbc/pom.xml
+++ b/tajo-jdbc/pom.xml
@@ -68,12 +68,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>compile</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <includeGroupIds>org.apache.tajo,joda-time</includeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -106,33 +105,28 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
+ <artifactId>hadoop-client</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index c345b73..df06ea0 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -39,6 +39,7 @@
<protobuf.version>2.5.0</protobuf.version>
<tajo.version>0.8.0-SNAPSHOT</tajo.version>
<tajo.root>${project.parent.relativePath}/..</tajo.root>
+ <extra.source.path>src/main/hadoop-${hadoop.version}</extra.source.path>
</properties>
<licenses>
@@ -589,7 +590,7 @@
</goals>
<configuration>
<sources>
- <source>src/main/hadoop-${hadoop.version}</source>
+ <source>${extra.source.path}</source>
</sources>
</configuration>
</execution>
@@ -601,7 +602,7 @@
</goals>
<configuration>
<sources>
- <source>src/main/hadoop-${hadoop.version}</source>
+ <source>${extra.source.path}</source>
</sources>
</configuration>
</execution>
@@ -696,6 +697,11 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
@@ -875,9 +881,14 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>15.0</version>
+ <version>11.0.2</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
@@ -900,6 +911,16 @@
<artifactId>joda-time</artifactId>
<version>2.3</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>10.8.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ <version>2.11</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<profiles>
@@ -974,6 +995,18 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>has-extra-source</id>
+ <activation>
+ <file>
+ <missing>src/main/hadoop-${hadoop.version}</missing>
+ </file>
+ </activation>
+ <properties>
+ <!-- default source path of tajo-0.8.0 -->
+ <extra.source.path>src/main/hadoop-2.3.0</extra.source.path>
+ </properties>
+ </profile>
</profiles>
<reporting>