You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2014/04/15 11:59:17 UTC
[1/2] TAJO-753: Clean up of maven dependencies. (jinho)
Repository: tajo
Updated Branches:
refs/heads/branch-0.8.0 031d5dd65 -> 646cf1ade
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-rpc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-rpc/pom.xml b/tajo-rpc/pom.xml
index 7f7ae79..839e3e0 100644
--- a/tajo-rpc/pom.xml
+++ b/tajo-rpc/pom.xml
@@ -137,54 +137,22 @@
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
</dependencies>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
index 214a4ba..b4e5f9a 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import com.google.protobuf.ServiceException;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.tajo.conf.TajoConf;
public abstract class ServerCallable<T> {
@@ -156,8 +155,8 @@ public abstract class ServerCallable<T> {
if (t instanceof UndeclaredThrowableException) {
t = t.getCause();
}
- if (t instanceof RemoteException) {
- t = ((RemoteException)t).unwrapRemoteException();
+ if (t instanceof RemoteException && t.getCause() != null) {
+ t = t.getCause();
}
return t;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml
index 5850ed4..ad81f71 100644
--- a/tajo-storage/pom.xml
+++ b/tajo-storage/pom.xml
@@ -155,13 +155,11 @@
<dependencies>
<dependency>
<groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
<artifactId>tajo-catalog-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-rpc</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
@@ -184,10 +182,10 @@
</exclusion>
</exclusions>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>zookeeper</artifactId>
@@ -206,6 +204,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-el</groupId>
@@ -229,7 +228,6 @@
</exclusion>
</exclusions>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
@@ -255,9 +253,32 @@
<groupId>com.sun.jersey.jersey-test-framework</groupId>
<artifactId>jersey-test-framework-grizzly2</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-hs</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
</exclusions>
</dependency>
-
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -268,31 +289,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
-
- <dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-column</artifactId>
<version>${parquet.version}</version>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
index 6615208..a2dda76 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
@@ -22,7 +22,6 @@ package org.apache.tajo.storage;
import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.commons.net.util.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -52,6 +51,7 @@ public abstract class AbstractStorageManager {
protected final FileSystem fs;
protected final Path tableBaseDir;
protected final boolean blocksMetadataEnabled;
+ private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0));
/**
* Cache of scanner handlers for each storage type.
@@ -492,15 +492,8 @@ public abstract class AbstractStorageManager {
int[] diskIds = new int[volumeIds.length];
for (int i = 0; i < volumeIds.length; i++) {
int diskId = -1;
- if (volumeIds[i] != null && volumeIds[i].isValid()) {
- String volumeIdString = volumeIds[i].toString();
- byte[] volumeIdBytes = Base64.decodeBase64(volumeIdString);
-
- if (volumeIdBytes.length == 4) {
- diskId = Bytes.toInt(volumeIdBytes);
- } else if (volumeIdBytes.length == 1) {
- diskId = (int) volumeIdBytes[0]; // support hadoop-2.0.2
- }
+ if (volumeIds[i] != null && volumeIds[i].hashCode() > 0) {
+ diskId = volumeIds[i].hashCode() - zeroVolumeId.hashCode();
}
diskIds[i] = diskId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index 857278a..652a8e9 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -82,10 +82,10 @@ public class CSVFile {
this.fs = path.getFileSystem(conf);
this.meta = meta;
this.schema = schema;
- this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(CatalogConstants.CSVFILE_DELIMITER,
- CatalogConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
+ this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
this.columnNum = schema.size();
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(CatalogConstants.CSVFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_NULL));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -108,7 +108,7 @@ public class CSVFile {
isShuffle = false;
}
- String codecName = this.meta.getOption(CatalogConstants.COMPRESSION_CODEC);
+ String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
if(!StringUtils.isEmpty(codecName)){
codecFactory = new CompressionCodecFactory(conf);
codec = codecFactory.getCodecByClassName(codecName);
@@ -139,7 +139,7 @@ public class CSVFile {
}
try {
- String serdeClass = this.meta.getOption(CatalogConstants.CSVFILE_SERDE,
+ String serdeClass = this.meta.getOption(StorageConstants.CSVFILE_SERDE,
TextSerializerDeserializer.class.getName());
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
} catch (Exception e) {
@@ -260,10 +260,10 @@ public class CSVFile {
}
//Delimiter
- String delim = meta.getOption(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ String delim = meta.getOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0);
- String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(CatalogConstants.CSVFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.CSVFILE_NULL));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -340,7 +340,7 @@ public class CSVFile {
}
try {
- String serdeClass = this.meta.getOption(CatalogConstants.CSVFILE_SERDE,
+ String serdeClass = this.meta.getOption(StorageConstants.CSVFILE_SERDE,
TextSerializerDeserializer.class.getName());
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/main/java/org/apache/tajo/storage/StorageConstants.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageConstants.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageConstants.java
new file mode 100644
index 0000000..414fa35
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageConstants.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage;
+
+import parquet.hadoop.ParquetWriter;
+import parquet.hadoop.metadata.CompressionCodecName;
+
+public class StorageConstants {
+ // table options
+ public static final String COMPRESSION_CODEC = "compression.codec";
+ public static final String COMPRESSION_TYPE = "compression.type";
+
+ public static final String CSVFILE_DELIMITER = "csvfile.delimiter";
+ public static final String CSVFILE_NULL = "csvfile.null";
+ public static final String CSVFILE_SERDE = "csvfile.serde";
+
+
+ public static final String SEQUENCEFILE_DELIMITER = "sequencefile.delimiter";
+ public static final String SEQUENCEFILE_NULL = "sequencefile.null";
+ public static final String SEQUENCEFILE_SERDE = "sequencefile.serde";
+
+ public static final String RCFILE_NULL = "rcfile.null";
+ public static final String RCFILE_SERDE = "rcfile.serde";
+
+ public static final String DEFAULT_FIELD_DELIMITER = "|";
+ public static final String DEFAULT_BINARY_SERDE = BinarySerializerDeserializer.class.getName();
+ public static final String DEFAULT_TEXT_SERDE = TextSerializerDeserializer.class.getName();
+
+ public static final String PARQUET_DEFAULT_BLOCK_SIZE;
+ public static final String PARQUET_DEFAULT_PAGE_SIZE;
+ public static final String PARQUET_DEFAULT_COMPRESSION_CODEC_NAME;
+ public static final String PARQUET_DEFAULT_IS_DICTIONARY_ENABLED;
+ public static final String PARQUET_DEFAULT_IS_VALIDATION_ENABLED;
+
+ static {
+ PARQUET_DEFAULT_BLOCK_SIZE =
+ Integer.toString(ParquetWriter.DEFAULT_BLOCK_SIZE);
+ PARQUET_DEFAULT_PAGE_SIZE =
+ Integer.toString(ParquetWriter.DEFAULT_PAGE_SIZE);
+
+ // When parquet-hadoop 1.3.3 is available, this should be changed to
+ // ParquetWriter.DEFAULT_COMPRESSION_CODEC_NAME.
+ PARQUET_DEFAULT_COMPRESSION_CODEC_NAME =
+ CompressionCodecName.UNCOMPRESSED.name().toLowerCase();
+
+ // When parquet-hadoop 1.3.3 is available, this should be changed to
+ // ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED.
+ PARQUET_DEFAULT_IS_DICTIONARY_ENABLED = "true";
+
+ // When parquet-hadoop 1.3.3 is available, this should be changed to
+ // ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED.
+ PARQUET_DEFAULT_IS_VALIDATION_ENABLED = "false";
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 1f13946..2acae5a 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -22,14 +22,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.util.FileUtil;
+import parquet.hadoop.ParquetOutputFormat;
import java.io.IOException;
-public class StorageUtil {
+public class StorageUtil extends StorageConstants{
public static int getRowByteSize(Schema schema) {
int sum = 0;
for(Column col : schema.getColumns()) {
@@ -99,4 +99,24 @@ public class StorageUtil {
return new Path(parent, sb.toString());
}
+
+ public static Options newPhysicalProperties(CatalogProtos.StoreType type) {
+ Options options = new Options();
+ if (CatalogProtos.StoreType.CSV == type) {
+ options.put(CSVFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ } else if (CatalogProtos.StoreType.RCFILE == type) {
+ options.put(RCFILE_SERDE, DEFAULT_BINARY_SERDE);
+ } else if (CatalogProtos.StoreType.SEQUENCEFILE == type) {
+ options.put(SEQUENCEFILE_SERDE, DEFAULT_TEXT_SERDE);
+ options.put(SEQUENCEFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ } else if (type == CatalogProtos.StoreType.PARQUET) {
+ options.put(ParquetOutputFormat.BLOCK_SIZE, PARQUET_DEFAULT_BLOCK_SIZE);
+ options.put(ParquetOutputFormat.PAGE_SIZE, PARQUET_DEFAULT_PAGE_SIZE);
+ options.put(ParquetOutputFormat.COMPRESSION, PARQUET_DEFAULT_COMPRESSION_CODEC_NAME);
+ options.put(ParquetOutputFormat.ENABLE_DICTIONARY, PARQUET_DEFAULT_IS_DICTIONARY_ENABLED);
+ options.put(ParquetOutputFormat.VALIDATION, PARQUET_DEFAULT_IS_VALIDATION_ENABLED);
+ }
+
+ return options;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
index 1beea99..1da41e4 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.io.*;
import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -42,8 +41,8 @@ import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.Bytes;
-import java.io.*;
import java.io.Closeable;
+import java.io.*;
import java.rmi.server.UID;
import java.security.MessageDigest;
import java.util.Arrays;
@@ -735,7 +734,7 @@ public class RCFile {
isShuffle = false;
}
- String codecClassname = this.meta.getOption(CatalogConstants.COMPRESSION_CODEC);
+ String codecClassname = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
if (!StringUtils.isEmpty(codecClassname)) {
try {
Class<? extends CompressionCodec> codecClass = conf.getClassByName(
@@ -747,7 +746,7 @@ public class RCFile {
}
}
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(CatalogConstants.RCFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.RCFILE_NULL));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -760,7 +759,7 @@ public class RCFile {
metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text("" + columnNumber));
- String serdeClass = this.meta.getOption(CatalogConstants.RCFILE_SERDE,
+ String serdeClass = this.meta.getOption(StorageConstants.RCFILE_SERDE,
BinarySerializerDeserializer.class.getName());
try {
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
@@ -768,7 +767,7 @@ public class RCFile {
LOG.error(e.getMessage(), e);
throw new IOException(e);
}
- metadata.set(new Text(CatalogConstants.RCFILE_SERDE), new Text(serdeClass));
+ metadata.set(new Text(StorageConstants.RCFILE_SERDE), new Text(serdeClass));
columnBuffers = new ColumnBuffer[columnNumber];
for (int i = 0; i < columnNumber; i++) {
@@ -1195,7 +1194,7 @@ public class RCFile {
rowId = new LongWritable();
readBytes = 0;
- String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(CatalogConstants.RCFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.RCFILE_NULL));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -1364,14 +1363,14 @@ public class RCFile {
metadata = new Metadata();
metadata.readFields(in);
- Text text = metadata.get(new Text(CatalogConstants.RCFILE_SERDE));
+ Text text = metadata.get(new Text(StorageConstants.RCFILE_SERDE));
try {
String serdeClass;
if(text != null && !text.toString().isEmpty()){
serdeClass = text.toString();
} else{
- serdeClass = this.meta.getOption(CatalogConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
+ serdeClass = this.meta.getOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
}
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
index 8092c0c..9eb1b2d 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -97,9 +96,10 @@ public class SequenceFileAppender extends FileAppender {
isShuffle = false;
}
- this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(CatalogConstants.SEQUENCEFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
+ this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
this.columnNum = schema.size();
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(CatalogConstants.SEQUENCEFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
@@ -110,7 +110,7 @@ public class SequenceFileAppender extends FileAppender {
throw new FileNotFoundException(path.toString());
}
- String codecName = this.meta.getOption(CatalogConstants.COMPRESSION_CODEC);
+ String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC);
if(!StringUtils.isEmpty(codecName)){
codecFactory = new CompressionCodecFactory(conf);
codec = codecFactory.getCodecByClassName(codecName);
@@ -121,7 +121,7 @@ public class SequenceFileAppender extends FileAppender {
}
try {
- String serdeClass = this.meta.getOption(CatalogConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
+ String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
@@ -135,7 +135,7 @@ public class SequenceFileAppender extends FileAppender {
valueClass = Text.class;
}
- String type = this.meta.getOption(CatalogConstants.COMPRESSION_TYPE, CompressionType.NONE.name());
+ String type = this.meta.getOption(StorageConstants.COMPRESSION_TYPE, CompressionType.NONE.name());
if (type.equals(CompressionType.BLOCK.name())) {
writer = SequenceFile.createWriter(fs, conf, path, BytesWritable.class, valueClass, CompressionType.BLOCK, codec);
} else if (type.equals(CompressionType.RECORD.name())) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
index c5720e3..ccf3d9e 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -88,14 +87,14 @@ public class SequenceFileScanner extends FileScanner {
reader = new SequenceFile.Reader(fs, fragment.getPath(), conf);
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(CatalogConstants.SEQUENCEFILE_NULL));
+ String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL));
if (StringUtils.isEmpty(nullCharacters)) {
nullChars = NullDatum.get().asTextBytes();
} else {
nullChars = nullCharacters.getBytes();
}
- String delim = meta.getOption(CatalogConstants.SEQUENCEFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ String delim = meta.getOption(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0);
this.start = fragment.getStartKey();
@@ -118,7 +117,7 @@ public class SequenceFileScanner extends FileScanner {
prepareProjection(targets);
try {
- String serdeClass = this.meta.getOption(CatalogConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
+ String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
if (serde instanceof BinarySerializerDeserializer)
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index 354fbc2..d50c356 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@ -94,7 +94,7 @@ public class TestMergeScanner {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.setOptions(CatalogUtil.newOptionsWithDefault(storeType));
+ meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
Path table1Path = new Path(testDir, storeType + "_1.data");
Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, table1Path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
index be8b6de..8d1d0b3 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
@@ -110,7 +110,7 @@ public class TestStorageManager {
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(1).build();
- int testCount = 100;
+ int testCount = 10;
Path tablePath = new Path("/testGetSplit");
try {
DistributedFileSystem fs = cluster.getFileSystem();
@@ -165,7 +165,7 @@ public class TestStorageManager {
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(2).build();
- int testCount = 100;
+ int testCount = 10;
Path tablePath = new Path("/testGetSplitWithBlockStorageLocationsBatching");
try {
DistributedFileSystem fs = cluster.getFileSystem();
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index a500f09..8a700ac 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -146,7 +146,7 @@ public class TestStorages {
schema.addColumn("score", Type.FLOAT4);
TableMeta meta = CatalogUtil.newTableMeta(storeType);
- meta.setOptions(CatalogUtil.newOptionsWithDefault(storeType));
+ meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
Path tablePath = new Path(testDir, "testProjection.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
@@ -209,7 +209,7 @@ public class TestStorages {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.setOptions(CatalogUtil.newOptionsWithDefault(storeType));
+ meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
@@ -271,11 +271,11 @@ public class TestStorages {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.setOptions(CatalogUtil.newOptionsWithDefault(storeType));
- meta.putOption(CatalogConstants.CSVFILE_NULL, "\\\\N");
- meta.putOption(CatalogConstants.RCFILE_NULL, "\\\\N");
- meta.putOption(CatalogConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName());
- meta.putOption(CatalogConstants.SEQUENCEFILE_NULL, "\\");
+ meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
+ meta.putOption(StorageConstants.CSVFILE_NULL, "\\\\N");
+ meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N");
+ meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName());
+ meta.putOption(StorageConstants.SEQUENCEFILE_NULL, "\\");
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
@@ -360,7 +360,7 @@ public class TestStorages {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(CatalogConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
+ meta.putOption(StorageConstants.CSVFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
@@ -429,7 +429,7 @@ public class TestStorages {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(CatalogConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
+ meta.putOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
@@ -498,7 +498,7 @@ public class TestStorages {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(CatalogConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
+ meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
@@ -567,7 +567,7 @@ public class TestStorages {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.putOption(CatalogConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName());
+ meta.putOption(StorageConstants.SEQUENCEFILE_SERDE, BinarySerializerDeserializer.class.getName());
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
index 140aa09..be5b096 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java
@@ -95,7 +95,7 @@ public class TestStorages {
schema.addColumn("age", Type.INT8);
TableMeta meta = CatalogUtil.newTableMeta(storeType);
- meta.setOptions(CatalogUtil.newOptionsWithDefault(storeType));
+ meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
Path tablePath = new Path(testDir, "Splitable.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
appender.enableStats();
@@ -148,7 +148,7 @@ public class TestStorages {
schema.addColumn("score", Type.FLOAT4);
TableMeta meta = CatalogUtil.newTableMeta(storeType);
- meta.setOptions(CatalogUtil.newOptionsWithDefault(storeType));
+ meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
Path tablePath = new Path(testDir, "testProjection.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
@@ -209,7 +209,7 @@ public class TestStorages {
Options options = new Options();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
- meta.setOptions(CatalogUtil.newOptionsWithDefault(storeType));
+ meta.setOptions(StorageUtil.newPhysicalProperties(storeType));
Path tablePath = new Path(testDir, "testVariousTypes.data");
Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
[2/2] git commit: TAJO-753: Clean up of maven dependencies. (jinho)
Posted by jh...@apache.org.
TAJO-753: Clean up of maven dependencies. (jinho)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/646cf1ad
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/646cf1ad
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/646cf1ad
Branch: refs/heads/branch-0.8.0
Commit: 646cf1ade76379f62d34657d032d6ee1ef8088f0
Parents: 031d5dd
Author: jinossy <ji...@gmail.com>
Authored: Tue Apr 15 18:58:00 2014 +0900
Committer: jinossy <ji...@gmail.com>
Committed: Tue Apr 15 18:58:00 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
tajo-algebra/pom.xml | 17 +-
tajo-catalog/tajo-catalog-client/pom.xml | 19 +-
tajo-catalog/tajo-catalog-common/pom.xml | 31 +--
.../apache/tajo/catalog/CatalogConstants.java | 49 ----
.../org/apache/tajo/catalog/CatalogUtil.java | 26 ---
tajo-catalog/tajo-catalog-drivers/pom.xml | 9 -
.../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 225 ++-----------------
.../tajo/catalog/store/HCatalogStore.java | 51 +++--
.../tajo/catalog/store/TestHCatalogStore.java | 37 +--
tajo-catalog/tajo-catalog-server/pom.xml | 26 ++-
tajo-client/pom.xml | 80 +------
tajo-common/pom.xml | 23 +-
.../org/apache/tajo/datum/TestInt2Datum.java | 6 +-
tajo-core/pom.xml | 1 +
tajo-core/tajo-core-backend/pom.xml | 77 ++-----
.../java/org/apache/tajo/benchmark/TPCH.java | 4 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 4 +-
.../tajo/engine/planner/LogicalPlanner.java | 5 +-
.../test/java/org/apache/tajo/TpchTestBase.java | 4 +-
.../org/apache/tajo/client/TestDDLBuilder.java | 9 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 8 +-
.../tajo/engine/query/TestNullValues.java | 12 +-
tajo-core/tajo-core-pullserver/pom.xml | 20 +-
.../pullserver/FadvisedFileRegionWrapper.java | 35 +++
.../pullserver/listener/FileCloseListener.java | 1 +
.../pullserver/FadvisedFileRegionWrapper.java | 34 +++
.../pullserver/FadvisedFileRegionWrapper.java | 36 +++
.../pullserver/listener/FileCloseListener.java | 42 ++++
.../tajo/pullserver/PullServerAuxService.java | 2 +-
.../tajo/pullserver/TajoPullServerService.java | 3 +-
tajo-jdbc/pom.xml | 30 +--
tajo-project/pom.xml | 39 +++-
tajo-rpc/pom.xml | 38 +---
.../org/apache/tajo/rpc/ServerCallable.java | 5 +-
tajo-storage/pom.xml | 64 +++---
.../tajo/storage/AbstractStorageManager.java | 13 +-
.../java/org/apache/tajo/storage/CSVFile.java | 16 +-
.../apache/tajo/storage/StorageConstants.java | 70 ++++++
.../org/apache/tajo/storage/StorageUtil.java | 28 ++-
.../org/apache/tajo/storage/rcfile/RCFile.java | 17 +-
.../sequencefile/SequenceFileAppender.java | 12 +-
.../sequencefile/SequenceFileScanner.java | 7 +-
.../apache/tajo/storage/TestMergeScanner.java | 2 +-
.../apache/tajo/storage/TestStorageManager.java | 4 +-
.../org/apache/tajo/storage/TestStorages.java | 22 +-
.../apache/tajo/storage/v2/TestStorages.java | 6 +-
47 files changed, 540 insertions(+), 731 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9f4dd22..6e1a178 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -629,6 +629,8 @@ Release 0.8.0 - unreleased
TASKS
+ TAJO-753: Clean up of maven dependencies. (jinho)
+
TAJO-529: Fix warnings in tajo-algebra. (jaehwa)
TAJO-700: Update site, wikis, pom.xml and other resources to point to the
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-algebra/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-algebra/pom.xml b/tajo-algebra/pom.xml
index 2f91b85..dea4727 100644
--- a/tajo-algebra/pom.xml
+++ b/tajo-algebra/pom.xml
@@ -68,12 +68,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <excludeGroupIds>org.apache.hadoop</excludeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -98,24 +97,10 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-client/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/pom.xml b/tajo-catalog/tajo-catalog-client/pom.xml
index b511739..66664d4 100644
--- a/tajo-catalog/tajo-catalog-client/pom.xml
+++ b/tajo-catalog/tajo-catalog-client/pom.xml
@@ -128,23 +128,14 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-rpc</artifactId>
</dependency>
-
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-common/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/pom.xml b/tajo-catalog/tajo-catalog-common/pom.xml
index fa90237..233079d 100644
--- a/tajo-catalog/tajo-catalog-common/pom.xml
+++ b/tajo-catalog/tajo-catalog-common/pom.xml
@@ -117,6 +117,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
@@ -139,10 +140,10 @@
<artifactId>tajo-common</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-rpc</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
-
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -152,30 +153,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>parquet-column</artifactId>
- <version>${parquet.version}</version>
- </dependency>
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>parquet-hadoop</artifactId>
- <version>${parquet.version}</version>
- </dependency>
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>parquet-format</artifactId>
- <version>${parquet.format.version}</version>
- </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
index 8cad4fe..d823f25 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
@@ -18,9 +18,6 @@
package org.apache.tajo.catalog;
-import parquet.hadoop.ParquetWriter;
-import parquet.hadoop.metadata.CompressionCodecName;
-
public class CatalogConstants {
public final static String IDENTIFIER_DELIMITER_REGEXP = "\\.";
public final static String IDENTIFIER_DELIMITER = ".";
@@ -54,50 +51,4 @@ public class CatalogConstants {
public static final String COL_DATABASES_PK = "DB_ID";
public static final String COL_TABLES_PK = "TID";
public static final String COL_TABLES_NAME = "TABLE_NAME";
-
- // table options
- public static final String COMPRESSION_CODEC = "compression.codec";
- public static final String COMPRESSION_TYPE = "compression.type";
-
- public static final String CSVFILE_DELIMITER = "csvfile.delimiter";
- public static final String CSVFILE_NULL = "csvfile.null";
- public static final String CSVFILE_SERDE = "csvfile.serde";
-
-
- public static final String SEQUENCEFILE_DELIMITER = "sequencefile.delimiter";
- public static final String SEQUENCEFILE_NULL = "sequencefile.null";
- public static final String SEQUENCEFILE_SERDE = "sequencefile.serde";
-
- public static final String RCFILE_NULL = "rcfile.null";
- public static final String RCFILE_SERDE = "rcfile.serde";
-
- public static final String DEFAULT_FIELD_DELIMITER = "|";
- public static final String DEFAULT_BINARY_SERDE = "org.apache.tajo.storage.BinarySerializerDeserializer";
- public static final String DEFAULT_TEXT_SERDE = "org.apache.tajo.storage.TextSerializerDeserializer";
-
- public static final String PARQUET_DEFAULT_BLOCK_SIZE;
- public static final String PARQUET_DEFAULT_PAGE_SIZE;
- public static final String PARQUET_DEFAULT_COMPRESSION_CODEC_NAME;
- public static final String PARQUET_DEFAULT_IS_DICTIONARY_ENABLED;
- public static final String PARQUET_DEFAULT_IS_VALIDATION_ENABLED;
-
- static {
- PARQUET_DEFAULT_BLOCK_SIZE =
- Integer.toString(ParquetWriter.DEFAULT_BLOCK_SIZE);
- PARQUET_DEFAULT_PAGE_SIZE =
- Integer.toString(ParquetWriter.DEFAULT_PAGE_SIZE);
-
- // When parquet-hadoop 1.3.3 is available, this should be changed to
- // ParquetWriter.DEFAULT_COMPRESSION_CODEC_NAME.
- PARQUET_DEFAULT_COMPRESSION_CODEC_NAME =
- CompressionCodecName.UNCOMPRESSED.name().toLowerCase();
-
- // When parquet-hadoop 1.3.3 is available, this should be changed to
- // ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED.
- PARQUET_DEFAULT_IS_DICTIONARY_ENABLED = "true";
-
- // When parquet-hadoop 1.3.3 is available, this should be changed to
- // ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED.
- PARQUET_DEFAULT_IS_VALIDATION_ENABLED = "false";
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 7555325..7568b26 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -36,8 +36,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import parquet.hadoop.ParquetOutputFormat;
-
import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import static org.apache.tajo.common.TajoDataTypes.Type;
@@ -293,30 +291,6 @@ public class CatalogUtil {
return null;
}
}
- public static Options newOptionsWithDefault(StoreType type) {
- Options options = new Options();
- if(StoreType.CSV == type){
- options.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
- } else if(StoreType.RCFILE == type){
- options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.DEFAULT_BINARY_SERDE);
- } else if(StoreType.SEQUENCEFILE == type){
- options.put(CatalogConstants.SEQUENCEFILE_SERDE, CatalogConstants.DEFAULT_TEXT_SERDE);
- options.put(CatalogConstants.SEQUENCEFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
- } else if (type == StoreType.PARQUET) {
- options.put(ParquetOutputFormat.BLOCK_SIZE,
- CatalogConstants.PARQUET_DEFAULT_BLOCK_SIZE);
- options.put(ParquetOutputFormat.PAGE_SIZE,
- CatalogConstants.PARQUET_DEFAULT_PAGE_SIZE);
- options.put(ParquetOutputFormat.COMPRESSION,
- CatalogConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME);
- options.put(ParquetOutputFormat.ENABLE_DICTIONARY,
- CatalogConstants.PARQUET_DEFAULT_IS_DICTIONARY_ENABLED);
- options.put(ParquetOutputFormat.VALIDATION,
- CatalogConstants.PARQUET_DEFAULT_IS_VALIDATION_ENABLED);
- }
-
- return options;
- }
public static TableMeta newTableMeta(StoreType type) {
return new TableMeta(type, new Options());
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/pom.xml b/tajo-catalog/tajo-catalog-drivers/pom.xml
index 157de68..97b8062 100644
--- a/tajo-catalog/tajo-catalog-drivers/pom.xml
+++ b/tajo-catalog/tajo-catalog-drivers/pom.xml
@@ -47,15 +47,6 @@
<profiles>
<profile>
- <id>hcatalog-0.11.0</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>tajo-hcatalog</module>
- </modules>
- </profile>
- <profile>
<id>hcatalog-0.12.0</id>
<activation>
<activeByDefault>false</activeByDefault>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
index 86ba343..e8f6855 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
@@ -57,6 +57,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
@@ -94,27 +95,14 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-rpc</artifactId>
</dependency>
-
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>10.8.2.2</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
@@ -128,189 +116,22 @@
<version>0.9.0</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<profiles>
<profile>
- <id>hcatalog-0.11.0</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <hive.version>0.11.0</hive.version>
- </properties>
- <dependencies>
- <dependency>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- <version>2.3-eb</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-contrib</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-hbase-handler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-testutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libfb303</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shimss</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libfb303</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hcatalog</groupId>
- <artifactId>hcatalog-core</artifactId>
- <version>${hive.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-service</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
- </profile>
- <profile>
<id>hcatalog-0.12.0</id>
<activation>
<activeByDefault>false</activeByDefault>
@@ -365,6 +186,10 @@
<groupId>com.jolbox</groupId>
<artifactId>bonecp</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -433,6 +258,10 @@
<groupId>com.jolbox</groupId>
<artifactId>bonecp</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -475,12 +304,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
- </dependency>
</dependencies>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 0fe255c..4880447 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -46,6 +46,7 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.exception.NotImplementedException;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.exception.InternalException;
+import org.apache.tajo.storage.StorageConstants;
import java.io.IOException;
import java.util.*;
@@ -190,24 +191,24 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat));
if (storeType.equals(CatalogProtos.StoreType.CSV)) {
- options.put(CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
} else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) {
- options.put(RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.put(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) {
- options.put(RCFILE_SERDE, DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (ColumnarSerDe.class.getName().equals(serde)) {
- options.put(RCFILE_SERDE, DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
} else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) {
- options.put(SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
- options.put(SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
+ options.put(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter));
+ options.put(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat));
String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB);
if (LazyBinarySerDe.class.getName().equals(serde)) {
- options.put(SEQUENCEFILE_SERDE, DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (LazySimpleSerDe.class.getName().equals(serde)) {
- options.put(SEQUENCEFILE_SERDE, DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
}
}
@@ -254,7 +255,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
}
} finally {
- client.release();
+ if(client != null) client.release();
}
TableMeta meta = new TableMeta(storeType, options);
@@ -288,7 +289,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
} catch (MetaException e) {
throw new CatalogException(e);
} finally {
- client.release();
+ if(client != null) client.release();
}
}
@@ -465,26 +466,27 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) {
- String serde = tableDesc.getMeta().getOption(RCFILE_SERDE);
+ String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName());
- if (DEFAULT_TEXT_SERDE.equals(serde)) {
+ if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName());
} else {
sd.getSerdeInfo().setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(RCFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) {
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
- StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(RCFILE_NULL)));
+ StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV)) {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName());
- String fieldDelimiter = tableDesc.getMeta().getOption(CSVFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
@@ -496,19 +498,20 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM,
StringEscapeUtils.unescapeJava(fieldDelimiter));
- if (tableDesc.getMeta().getOption(CSVFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) {
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
- StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(CSVFILE_NULL)));
+ StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL)));
}
} else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE)) {
- String serde = tableDesc.getMeta().getOption(SEQUENCEFILE_SERDE);
+ String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE);
sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName());
sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName());
- if (DEFAULT_TEXT_SERDE.equals(serde)) {
+ if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
- String fieldDelimiter = tableDesc.getMeta().getOption(SEQUENCEFILE_DELIMITER, DEFAULT_FIELD_DELIMITER);
+ String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER);
// User can use an unicode for filed delimiter such as \u0001, \001.
// In this case, java console will convert this value into "\\u001".
@@ -523,9 +526,9 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName());
}
- if (tableDesc.getMeta().getOption(SEQUENCEFILE_NULL) != null) {
+ if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) {
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT,
- StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(SEQUENCEFILE_NULL)));
+ StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL)));
}
} else {
throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType().name()));
@@ -540,7 +543,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
} catch (Exception e) {
throw new CatalogException(e);
} finally {
- client.release();
+ if(client != null) client.release();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index f4ca4d1..0cb4520 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -29,6 +29,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.CommonTestingUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -104,15 +105,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(StringEscapeUtils.escapeJava(CatalogConstants.DEFAULT_FIELD_DELIMITER),
- table1.getMeta().getOption(CatalogConstants.CSVFILE_DELIMITER));
+ assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER),
+ table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER));
store.dropTable(DB_NAME, CUSTOMER);
}
@Test
public void testTableUsingRCFileWithBinarySerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -133,15 +134,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_BINARY_SERDE,
- table1.getMeta().getOption(CatalogConstants.RCFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_BINARY_SERDE,
+ table1.getMeta().getOption(StorageConstants.RCFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
@Test
public void testTableUsingRCFileWithTextSerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.RCFILE_SERDE, CatalogConstants.DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -162,15 +163,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(CatalogConstants.RCFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.RCFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
@Test
public void testTableWithNullValue() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001"));
- options.put(CatalogConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N"));
+ options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001"));
+ options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N"));
TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options);
@@ -198,11 +199,11 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(table.getMeta().getOption(CatalogConstants.CSVFILE_DELIMITER),
- table1.getMeta().getOption(CatalogConstants.CSVFILE_DELIMITER));
+ assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER),
+ table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER));
- assertEquals(table.getMeta().getOption(CatalogConstants.CSVFILE_NULL),
- table1.getMeta().getOption(CatalogConstants.CSVFILE_NULL));
+ assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_NULL),
+ table1.getMeta().getOption(StorageConstants.CSVFILE_NULL));
store.dropTable(DB_NAME, SUPPLIER);
}
@@ -304,7 +305,7 @@ public class TestHCatalogStore {
@Test
public void testTableUsingSequenceFileWithBinarySerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.SEQUENCEFILE_SERDE, CatalogConstants.DEFAULT_BINARY_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -325,15 +326,15 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_BINARY_SERDE,
- table1.getMeta().getOption(CatalogConstants.SEQUENCEFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_BINARY_SERDE,
+ table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
@Test
public void testTableUsingSequenceFileWithTextSerde() throws Exception {
Options options = new Options();
- options.put(CatalogConstants.SEQUENCEFILE_SERDE, CatalogConstants.DEFAULT_TEXT_SERDE);
+ options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE);
TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options);
org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
@@ -354,7 +355,7 @@ public class TestHCatalogStore {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}
- assertEquals(CatalogConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(CatalogConstants.SEQUENCEFILE_SERDE));
+ assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE));
store.dropTable(DB_NAME, REGION);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-catalog/tajo-catalog-server/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml
index 03e92ec..dfbec6d 100644
--- a/tajo-catalog/tajo-catalog-server/pom.xml
+++ b/tajo-catalog/tajo-catalog-server/pom.xml
@@ -94,6 +94,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
@@ -127,7 +128,21 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-rpc</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>zookeeper</artifactId>
+ <groupId>org.apache.zookeeper</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -138,17 +153,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
- <version>10.8.2.2</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-client/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-client/pom.xml b/tajo-client/pom.xml
index c6fbb27..6a14382 100644
--- a/tajo-client/pom.xml
+++ b/tajo-client/pom.xml
@@ -157,12 +157,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <excludeGroupIds>org.apache.hadoop</excludeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -180,20 +179,6 @@
<dependency>
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- <exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tajo</groupId>
@@ -210,59 +195,28 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
@@ -273,18 +227,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
-
- <dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
- <version>2.11</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-common/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml
index 00e2448..35ac3e5 100644
--- a/tajo-common/pom.xml
+++ b/tajo-common/pom.xml
@@ -129,19 +129,15 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<dependencies>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
@@ -150,20 +146,25 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<artifactId>commons-logging-api</artifactId>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <type>jar</type>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
- <type>jar</type>
- <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
index 832f563..6af9700 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestInt2Datum.java
@@ -18,12 +18,10 @@
package org.apache.tajo.datum;
-import org.junit.Test;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.junit.Test;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class TestInt2Datum {
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml
index e131d0e..709390d 100644
--- a/tajo-core/pom.xml
+++ b/tajo-core/pom.xml
@@ -119,6 +119,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<excludeGroupIds>org.apache.tajo</excludeGroupIds>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/pom.xml b/tajo-core/tajo-core-backend/pom.xml
index 86ea4daf..7e43bac 100644
--- a/tajo-core/tajo-core-backend/pom.xml
+++ b/tajo-core/tajo-core-backend/pom.xml
@@ -171,12 +171,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>runtime</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <excludeGroupIds>org.apache.hadoop</excludeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -235,6 +234,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -293,19 +293,22 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <scope>compile</scope>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -315,6 +318,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -333,75 +337,29 @@
<artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4</artifactId>
</dependency>
<dependency>
- <groupId>org.antlr</groupId>
- <artifactId>antlr4-runtime</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>10.8.2.2</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </dependency>
-
<dependency>
<groupId>org.dspace.dependencies</groupId>
<artifactId>dspace-geoip</artifactId>
<version>1.2.3</version>
</dependency>
-
<dependency>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
<version>3.1.1</version>
</dependency>
-
-
- <dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- <version>2.5-20081211</version>
- </dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
@@ -409,21 +367,10 @@
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- <version>6.1.14</version>
- </dependency>
- <dependency>
- <groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
<version>6.1.14</version>
</dependency>
<dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-api-2.1</artifactId>
- <version>6.1.14</version>
- </dependency>
-
- <dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${metrics.version}</version>
@@ -660,6 +607,10 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
index 5dd0b12..abd9e7f 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/benchmark/TPCH.java
@@ -23,12 +23,12 @@ import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.storage.StorageConstants;
import java.io.IOException;
import java.sql.SQLException;
@@ -189,7 +189,7 @@ public class TPCH extends BenchmarkSet {
private void loadTable(String tableName) throws ServiceException {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
- meta.putOption(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ meta.putOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
try {
tajo.createExternalTable(tableName, getSchema(tableName), new Path(dataDir, tableName), meta);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index f28f7ca..5d75edd 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -28,9 +28,9 @@ import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tajo.algebra.*;
import org.apache.tajo.algebra.Aggregation.GroupType;
import org.apache.tajo.algebra.LiteralValue.LiteralType;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.engine.parser.SQLParser.*;
+import org.apache.tajo.storage.StorageConstants;
import java.nio.charset.Charset;
import java.util.*;
@@ -1324,7 +1324,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
public Map<String, String> escapeTableMeta(Map<String, String> map) {
Map<String, String> params = new HashMap<String, String>();
for (Map.Entry<String, String> entry : map.entrySet()) {
- if (entry.getKey().equals(CatalogConstants.CSVFILE_DELIMITER)) {
+ if (entry.getKey().equals(StorageConstants.CSVFILE_DELIMITER)) {
params.put(entry.getKey(), escapeDelimiter(entry.getValue()));
} else {
params.put(entry.getKey(), entry.getValue());
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 8946c02..f2ddf13 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -42,6 +42,7 @@ import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule;
import org.apache.tajo.engine.utils.SchemaUtil;
import org.apache.tajo.master.session.Session;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -1321,8 +1322,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setStorageType(CatalogProtos.StoreType.CSV);
}
- // Set default options to be created.
- Options options = CatalogUtil.newOptionsWithDefault(createTableNode.getStorageType());
+ // Set default storage properties to be created.
+ Options options = StorageUtil.newPhysicalProperties(createTableNode.getStorageType());
if (expr.hasParams()) {
options.putAll(expr.getParams());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
index 8487e84..dd21385 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TpchTestBase.java
@@ -22,9 +22,9 @@ import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.benchmark.TPCH;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.FileUtil;
import java.io.File;
@@ -91,7 +91,7 @@ public class TpchTestBase {
private void setUp() throws Exception {
util = new LocalTajoTestingUtility();
Options opt = new Options();
- opt.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ opt.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
util.setup(names, paths, schemas, opt);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
index 9d8d631..1855217 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestDDLBuilder.java
@@ -24,12 +24,11 @@ import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.FileUtil;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
public class TestDDLBuilder {
@@ -43,8 +42,8 @@ public class TestDDLBuilder {
schema1.addColumn("addr", TajoDataTypes.Type.TEXT);
meta1 = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV);
- meta1.putOption(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
- meta1.putOption(CatalogConstants.COMPRESSION_CODEC, GzipCodec.class.getName());
+ meta1.putOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName());
Schema expressionSchema = new Schema();
expressionSchema.addColumn("key", TajoDataTypes.Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index 4dc5728..a4e31e0 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -26,9 +26,13 @@ import org.apache.tajo.IntegrationTest;
import org.apache.tajo.QueryTestCaseBase;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.CatalogService;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Options;
+import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.storage.StorageConstants;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -207,7 +211,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
Options options = desc.getMeta().getOptions();
assertNotNull(options);
- assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(CatalogConstants.CSVFILE_DELIMITER));
+ assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(StorageConstants.CSVFILE_DELIMITER));
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
index c460bcf..c5afc3d 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNullValues.java
@@ -20,10 +20,10 @@ package org.apache.tajo.engine.query;
import org.apache.tajo.IntegrationTest;
import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.Options;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.storage.StorageConstants;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -52,7 +52,7 @@ public class TestNullValues {
"3|filled|0.2"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ opts.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable1 where col3 is null");
@@ -78,7 +78,7 @@ public class TestNullValues {
"3|filled|"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, CatalogConstants.DEFAULT_FIELD_DELIMITER);
+ opts.put(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable2 where col1 is not null");
@@ -113,7 +113,7 @@ public class TestNullValues {
",,,43578,19,13,6,3581,2557,1024"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, ",");
+ opts.put(StorageConstants.CSVFILE_DELIMITER, ",");
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable3 where col1 is null and col2 is null and col3 is null and col4 = 43578");
@@ -146,8 +146,8 @@ public class TestNullValues {
",\\N,,43578"
};
Options opts = new Options();
- opts.put(CatalogConstants.CSVFILE_DELIMITER, ",");
- opts.put(CatalogConstants.CSVFILE_NULL, "\\\\N");
+ opts.put(StorageConstants.CSVFILE_DELIMITER, ",");
+ opts.put(StorageConstants.CSVFILE_NULL, "\\\\N");
ResultSet res = TajoTestingCluster
.run(table, schemas, opts, new String[][]{data},
"select * from nulltable4 where col1 is null and col2 is null and col3 is null and col5 is null and col4 = 43578");
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/pom.xml b/tajo-core/tajo-core-pullserver/pom.xml
index b5049be..cdc5ff6 100644
--- a/tajo-core/tajo-core-pullserver/pom.xml
+++ b/tajo-core/tajo-core-pullserver/pom.xml
@@ -34,6 +34,10 @@
<dependencies>
<dependency>
<groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-rpc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
<artifactId>tajo-catalog-common</artifactId>
<scope>provided</scope>
</dependency>
@@ -42,7 +46,6 @@
<artifactId>tajo-storage</artifactId>
<scope>provided</scope>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
@@ -59,21 +62,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging-api</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
new file mode 100644
index 0000000..335dff0
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver;
+
+import org.apache.hadoop.io.ReadaheadPool;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+@Deprecated
+public class FadvisedFileRegionWrapper extends org.apache.hadoop.mapred.FadvisedFileRegion {
+
+
+ public FadvisedFileRegionWrapper(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier) throws IOException {
+ super(file, position, count, manageOsCache, readaheadLength, readaheadPool, identifier);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
index b7f3e34..0933167 100644
--- a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.2.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.mapred.FadvisedFileRegion;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
+@Deprecated
public class FileCloseListener implements ChannelFutureListener {
private FadvisedFileRegion filePart;
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
new file mode 100644
index 0000000..42d1cd8
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.3.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver;
+
+import org.apache.hadoop.io.ReadaheadPool;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+public class FadvisedFileRegionWrapper extends org.apache.hadoop.mapred.FadvisedFileRegion {
+
+
+ public FadvisedFileRegionWrapper(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier) throws IOException {
+ super(file, position, count, manageOsCache, readaheadLength, readaheadPool, identifier);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
new file mode 100644
index 0000000..ee53bc6
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/FadvisedFileRegionWrapper.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver;
+
+import org.apache.hadoop.io.ReadaheadPool;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/* tajo is not yet supported on Windows */
+public class FadvisedFileRegionWrapper extends org.apache.hadoop.mapred.FadvisedFileRegion {
+ public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;
+
+ public FadvisedFileRegionWrapper(RandomAccessFile file, long position, long count,
+ boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
+ String identifier) throws IOException {
+ super(file, position, count, manageOsCache, readaheadLength, readaheadPool,
+ identifier, DEFAULT_SHUFFLE_BUFFER_SIZE, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
new file mode 100644
index 0000000..5b2d1b3
--- /dev/null
+++ b/tajo-core/tajo-core-pullserver/src/main/hadoop-2.4.0/org/apache/tajo/pullserver/listener/FileCloseListener.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.pullserver.listener;
+
+import org.apache.hadoop.mapred.FadvisedFileRegion;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+
+public class FileCloseListener implements ChannelFutureListener {
+
+ private FadvisedFileRegion filePart;
+
+ public FileCloseListener(FadvisedFileRegion filePart) {
+ this.filePart = filePart;
+ }
+
+ // TODO error handling; distinguish IO/connection failures,
+ // attribute to appropriate spill output
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if(future.isSuccess()){
+ filePart.transferSuccessful();
+ }
+ filePart.releaseExternalResources();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
index 40978fe..87011da 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
@@ -490,7 +490,7 @@ public class PullServerAuxService extends AuxiliaryService {
}
ChannelFuture writeFuture;
if (ch.getPipeline().get(SslHandler.class) == null) {
- final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
+ final FadvisedFileRegionWrapper partition = new FadvisedFileRegionWrapper(spill,
file.startOffset, file.length(), manageOsCache, readaheadLength,
readaheadPool, file.getFile().getAbsolutePath());
writeFuture = ch.write(partition);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
index 70f5421..cc3cb2e 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.mapred.FadvisedChunkedFile;
-import org.apache.hadoop.mapred.FadvisedFileRegion;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -487,7 +486,7 @@ public class TajoPullServerService extends AbstractService {
}
ChannelFuture writeFuture;
if (ch.getPipeline().get(SslHandler.class) == null) {
- final FadvisedFileRegion filePart = new FadvisedFileRegion(spill,
+ final FadvisedFileRegionWrapper filePart = new FadvisedFileRegionWrapper(spill,
file.startOffset, file.length(), manageOsCache, readaheadLength,
readaheadPool, file.getFile().getAbsolutePath());
writeFuture = ch.write(filePart);
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml
index d691af7..7a615a0 100644
--- a/tajo-jdbc/pom.xml
+++ b/tajo-jdbc/pom.xml
@@ -68,12 +68,11 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
+ <includeScope>compile</includeScope>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <includeGroupIds>org.apache.tajo,joda-time</includeGroupIds>
- <excludeScope>provided</excludeScope>
</configuration>
</execution>
</executions>
@@ -106,33 +105,28 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
+ <artifactId>hadoop-client</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
+ <artifactId>protobuf-java</artifactId>
+ <groupId>com.google.protobuf</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
</exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/tajo/blob/646cf1ad/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index c345b73..df06ea0 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -39,6 +39,7 @@
<protobuf.version>2.5.0</protobuf.version>
<tajo.version>0.8.0-SNAPSHOT</tajo.version>
<tajo.root>${project.parent.relativePath}/..</tajo.root>
+ <extra.source.path>src/main/hadoop-${hadoop.version}</extra.source.path>
</properties>
<licenses>
@@ -589,7 +590,7 @@
</goals>
<configuration>
<sources>
- <source>src/main/hadoop-${hadoop.version}</source>
+ <source>${extra.source.path}</source>
</sources>
</configuration>
</execution>
@@ -601,7 +602,7 @@
</goals>
<configuration>
<sources>
- <source>src/main/hadoop-${hadoop.version}</source>
+ <source>${extra.source.path}</source>
</sources>
</configuration>
</execution>
@@ -696,6 +697,11 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
@@ -875,9 +881,14 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>15.0</version>
+ <version>11.0.2</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
@@ -900,6 +911,16 @@
<artifactId>joda-time</artifactId>
<version>2.3</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>10.8.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ <version>2.11</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<profiles>
@@ -974,6 +995,18 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>has-extra-source</id>
+ <activation>
+ <file>
+ <missing>src/main/hadoop-${hadoop.version}</missing>
+ </file>
+ </activation>
+ <properties>
+ <!-- default source path of tajo-0.8.0 -->
+ <extra.source.path>src/main/hadoop-2.3.0</extra.source.path>
+ </properties>
+ </profile>
</profiles>
<reporting>