You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by gr...@apache.org on 2014/06/25 10:18:38 UTC
git commit: CRUNCH-428 Disallow empty list of paths for inputs
Repository: crunch
Updated Branches:
refs/heads/apache-crunch-0.8 31c7b6d8a -> 5b82e2612
CRUNCH-428 Disallow empty list of paths for inputs
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/5b82e261
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/5b82e261
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/5b82e261
Branch: refs/heads/apache-crunch-0.8
Commit: 5b82e2612dc621bad83b08e0fd1c79c6f444351d
Parents: 31c7b6d
Author: Gabriel Reid <gr...@apache.org>
Authored: Wed Jun 25 10:10:13 2014 +0200
Committer: Gabriel Reid <gr...@apache.org>
Committed: Wed Jun 25 10:12:40 2014 +0200
----------------------------------------------------------------------
.../main/java/org/apache/crunch/io/From.java | 8 ++--
.../apache/crunch/io/impl/FileSourceImpl.java | 15 +++---
.../apache/crunch/io/text/TextFileSource.java | 7 ++-
.../java/org/apache/crunch/io/FromTest.java | 48 ++++++++++++++++++++
4 files changed, 63 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/From.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/From.java b/crunch-core/src/main/java/org/apache/crunch/io/From.java
index d72ddf4..3c892a6 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/From.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/From.java
@@ -17,6 +17,10 @@
*/
package org.apache.crunch.io;
+import java.io.IOException;
+import java.util.List;
+
+import com.google.common.base.Preconditions;
import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.generic.GenericData;
@@ -45,9 +49,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import java.io.IOException;
-import java.util.List;
-
/**
* <p>Static factory methods for creating common {@link Source} types.</p>
*
@@ -311,6 +312,7 @@ public class From {
* @return A new {@code Source<GenericData.Record>} instance
*/
public static Source<GenericData.Record> avroFile(List<Path> paths, Configuration conf) {
+ Preconditions.checkArgument(!paths.isEmpty(), "At least one path must be supplied");
return avroFile(paths, Avros.generics(getSchemaFromPath(paths.get(0), conf)));
}
http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
index 1151ad5..9917bad 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileSourceImpl.java
@@ -17,11 +17,12 @@
*/
package org.apache.crunch.io.impl;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
import java.io.IOException;
-
import java.util.List;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,7 +40,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
public class FileSourceImpl<T> implements Source<T> {
@@ -64,7 +64,8 @@ public class FileSourceImpl<T> implements Source<T> {
}
public FileSourceImpl(List<Path> paths, PType<T> ptype, FormatBundle<? extends InputFormat> inputBundle) {
- this.path = paths.isEmpty() ? null : paths.get(0);
+ Preconditions.checkArgument(!paths.isEmpty(), "Must supply at least one input path");
+ this.path = paths.get(0);
this.paths = paths;
this.ptype = ptype;
this.inputBundle = inputBundle;
@@ -72,9 +73,7 @@ public class FileSourceImpl<T> implements Source<T> {
@Deprecated
public Path getPath() {
- if (paths.isEmpty()) {
- return null;
- } else if (paths.size() > 1) {
+ if (paths.size() > 1) {
LOG.warn("getPath() called for source with multiple paths, only " +
"returning first. Source: " + this);
}
http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java b/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
index 732288d..214ab78 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/text/TextFileSource.java
@@ -18,13 +18,12 @@
package org.apache.crunch.io.text;
import java.io.IOException;
-
import java.util.Collections;
import java.util.List;
+import org.apache.crunch.ReadableData;
import org.apache.crunch.impl.mr.run.RuntimeParameters;
import org.apache.crunch.io.ReadableSource;
-import org.apache.crunch.ReadableData;
import org.apache.crunch.io.impl.FileSourceImpl;
import org.apache.crunch.types.PType;
import org.apache.crunch.types.avro.AvroTypeFamily;
@@ -36,7 +35,7 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
public class TextFileSource<T> extends FileSourceImpl<T> implements ReadableSource<T> {
- private static <S> Class<? extends FileInputFormat<?, ?>> getInputFormat(Path path, PType<S> ptype) {
+ private static <S> Class<? extends FileInputFormat<?, ?>> getInputFormat(PType<S> ptype) {
if (ptype.getFamily().equals(AvroTypeFamily.getInstance())) {
return AvroUtf8InputFormat.class;
} else {
@@ -49,7 +48,7 @@ public class TextFileSource<T> extends FileSourceImpl<T> implements ReadableSour
}
public TextFileSource(List<Path> paths, PType<T> ptype) {
- super(paths, ptype, getInputFormat(paths.get(0), ptype));
+ super(paths, ptype, getInputFormat(ptype));
inputBundle.set(RuntimeParameters.DISABLE_COMBINE_FILE, Boolean.FALSE.toString());
}
http://git-wip-us.apache.org/repos/asf/crunch/blob/5b82e261/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java b/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java
new file mode 100644
index 0000000..06ef6cd
--- /dev/null
+++ b/crunch-core/src/test/java/org/apache/crunch/io/FromTest.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.io;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.junit.Test;
+
+public class FromTest {
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testAvroFile_EmptyPathListNotAllowed() {
+ From.avroFile(ImmutableList.<Path>of());
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testTextFile_EmptyPathListNotAllowed() {
+ From.textFile(ImmutableList.<Path>of());
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testFormattedFile_EmptyPathListNotAllowed() {
+ From.formattedFile(ImmutableList.<Path>of(), TextInputFormat.class, LongWritable.class, Text.class);
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testSequenceFile_EmptyPathListNotAllowed() {
+ From.sequenceFile(ImmutableList.<Path>of(), LongWritable.class, Text.class);
+ }
+}
\ No newline at end of file