You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/03/27 07:45:46 UTC
[7/8] git commit: DRILL-431: MapR profile - unit tests try to use
localhost as the MapR cluster
DRILL-431: MapR profile - unit tests try to use localhost as the MapR cluster
This was caused by incorrect configuration key: "fs.name.default", but should be "fs.default.name"
Replaced all occurrences of both with variable FileSystem.FS_DEFAULT_NAME_KEY
also updated hadoo-core version number for mapr profile
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a43db154
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a43db154
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a43db154
Branch: refs/heads/master
Commit: a43db154f6e9b3efc84a546dd01cbdc9c19f27a1
Parents: 1e25c0d
Author: Steven Phillips <sp...@maprtech.com>
Authored: Mon Mar 24 15:20:09 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Mar 26 23:39:39 2014 -0700
----------------------------------------------------------------------
.../drill/exec/physical/impl/trace/TraceRecordBatch.java | 2 +-
.../org/apache/drill/exec/store/dfs/FileSystemPlugin.java | 3 ++-
.../drill/exec/work/batch/SpoolingRawBatchBuffer.java | 3 +--
.../java/org/apache/drill/exec/cache/TestWriteToDisk.java | 2 +-
.../test/java/org/apache/drill/exec/client/DumpCatTest.java | 2 +-
.../drill/exec/physical/impl/trace/TestTraceOutputDump.java | 2 +-
.../apache/drill/exec/store/hive/HiveTestDataGenerator.java | 3 ++-
.../apache/drill/exec/store/parquet/TestFileGenerator.java | 2 +-
.../java/org/apache/drill/exec/work/batch/FileTest.java | 2 +-
pom.xml | 9 ++-------
10 files changed, 13 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java
index c9a73f9..88667ea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java
@@ -77,7 +77,7 @@ public class TraceRecordBatch extends AbstractSingleRecordBatch<Trace> {
/* Create the log file we will dump to and initialize the file descriptors */
try {
Configuration conf = new Configuration();
- conf.set("fs.name.default", ExecConstants.TRACE_DUMP_FILESYSTEM);
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, context.getConfig().getString(ExecConstants.TRACE_DUMP_FILESYSTEM));
FileSystem fs = FileSystem.get(conf);
/* create the file */
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index 3762c1a..d25d501 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration;
import com.beust.jcommander.internal.Lists;
import com.beust.jcommander.internal.Maps;
+import org.apache.hadoop.fs.FileSystem;
/**
* A Storage engine associated with a Hadoop FileSystem Implementation. Examples include HDFS, MapRFS, QuantacastFileSystem,
@@ -61,7 +62,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
this.context = context;
Configuration fsConf = new Configuration();
- fsConf.set("fs.default.name", config.connection);
+ fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection);
fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
this.fs = FileSystemCreator.getFileSystem(context.getConfig(), fsConf);
this.formatsByName = FormatCreator.getFormatPlugins(context, fs, config);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
index d7c407b..afac86f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java
@@ -53,7 +53,6 @@ import com.google.common.collect.Queues;
public class SpoolingRawBatchBuffer implements RawBatchBuffer {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SpoolingRawBatchBuffer.class);
- private static String HADOOP_FILESYSTEM_DEFAULT_NAME = "fs.default.name";
private static String DRILL_LOCAL_IMPL_STRING = "fs.drill-local.impl";
private static final float STOP_SPOOLING_FRACTION = (float) 0.5;
@@ -72,7 +71,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer {
this.context = context;
this.threshold = context.getConfig().getLong(ExecConstants.SPOOLING_BUFFER_MEMORY);
Configuration conf = new Configuration();
- conf.set(HADOOP_FILESYSTEM_DEFAULT_NAME, context.getConfig().getString(ExecConstants.TEMP_FILESYSTEM));
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, context.getConfig().getString(ExecConstants.TEMP_FILESYSTEM));
conf.set(DRILL_LOCAL_IMPL_STRING, LocalSyncableFileSystem.class.getName());
this.fs = FileSystem.get(conf);
this.path = new Path(getDir(), getFileName());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
index fb3e821..52e8425 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java
@@ -75,7 +75,7 @@ public class TestWriteToDisk {
VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, context.getAllocator());
Configuration conf = new Configuration();
- conf.set("fs.name.default", "file:///");
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("/tmp/drillSerializable");
if (fs.exists(path)) fs.delete(path, false);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
index c513cd0..9545845 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java
@@ -103,7 +103,7 @@ public class DumpCatTest {
System.out.println("File Name: " + filename);
Configuration conf = new Configuration();
- conf.set("fs.name.default", c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));
FileSystem fs = FileSystem.get(conf);
Path path = new Path(filename);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
index f166fad..910d782 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java
@@ -114,7 +114,7 @@ public class TestTraceOutputDump {
System.out.println("File Name: " + filename);
Configuration conf = new Configuration();
- conf.set("fs.name.default", c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM));
FileSystem fs = FileSystem.get(conf);
Path path = new Path(filename);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 87b14ed..8a71e2a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
@@ -59,7 +60,7 @@ public class HiveTestDataGenerator {
HiveConf conf = new HiveConf();
conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true", DB_DIR));
- conf.set("fs.default.name", "file:///");
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
conf.set("hive.metastore.warehouse.dir", WH_DIR);
String tableName = "kv";
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
index d8892dc..2d2a2ec 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
@@ -108,7 +108,7 @@ public class TestFileGenerator {
WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
Configuration configuration = new Configuration();
- configuration.set("fs.default.name", "file:///");
+ configuration.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
//"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
FileSystem fs = FileSystem.get(configuration);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
index 4406e04..6248f57 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java
@@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit;
public class FileTest {
public static void main(String[] args) throws IOException {
Configuration conf = new Configuration();
- conf.set("fs.default.name", "sync:///");
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "sync:///");
System.out.println(FileSystem.getDefaultUri(conf));
FileSystem fs = FileSystem.get(conf);
// FileSystem fs = new LocalSyncableFileSystem(conf);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 76523bf..5998f99 100644
--- a/pom.xml
+++ b/pom.xml
@@ -608,7 +608,7 @@
<id>mapr</id>
<properties>
<alt-hadoop>mapr</alt-hadoop>
- <rat.excludeSubprojects>false</rat.excludeSubprojects>
+ <rat.excludeSubprojects>true</rat.excludeSubprojects>
</properties>
<dependencyManagement>
<dependencies>
@@ -623,11 +623,6 @@
<version>2.7.1</version>
</dependency>
<dependency>
- <groupId>com.mapr.hadoop</groupId>
- <artifactId>maprfs</artifactId>
- <version>1.0.3-mapr-3.0.0</version>
- </dependency>
- <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
<version>0.94.13-mapr-1401-m7-3.0.2</version>
@@ -641,7 +636,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
- <version>1.0.3-mapr-3.0.0</version>
+ <version>1.0.3-mapr-3.0.2</version>
<exclusions>
<exclusion>
<artifactId>asm</artifactId>