You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by sm...@apache.org on 2017/04/16 21:24:28 UTC
opennlp git commit: OPENNLP-1024:Add unit tests and javadocs for
DirectorySampleStream, this closes apache/opennlp#160
Repository: opennlp
Updated Branches:
refs/heads/master cff6e0009 -> 27214015c
OPENNLP-1024:Add unit tests and javadocs for DirectorySampleStream, this closes apache/opennlp#160
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/27214015
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/27214015
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/27214015
Branch: refs/heads/master
Commit: 27214015c725619e0098e227fd29816d0891d4c5
Parents: cff6e00
Author: jzonthemtn <je...@mtnfog.com>
Authored: Sun Apr 16 17:24:16 2017 -0400
Committer: smarthi <sm...@apache.org>
Committed: Sun Apr 16 17:24:19 2017 -0400
----------------------------------------------------------------------
.../tools/formats/DirectorySampleStream.java | 34 +++-
.../formats/DirectorySampleStreamTest.java | 187 +++++++++++++++++++
2 files changed, 215 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/27214015/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java
index 3a5621a..da73507 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java
@@ -28,24 +28,30 @@ import java.util.Stack;
import opennlp.tools.util.ObjectStream;
/**
- * The directory sample stream scans a directory (recursively) for plain text
- * files and outputs each file as a String object.
+ * The directory sample stream allows for creating a stream
+ * from a directory listing of files.
*/
public class DirectorySampleStream implements ObjectStream<File> {
private final List<File> inputDirectories;
- private final boolean isRecursiveScan;
+ private final boolean recursive;
private final FileFilter fileFilter;
private Stack<File> directories = new Stack<>();
private Stack<File> textFiles = new Stack<>();
-
+
+ /**
+ * Creates a new directory sample stream.
+ * @param dirs The directories to read.
+ * @param fileFilter The {@link FileFilter filter} to apply while enumerating files.
+ * @param recursive Enables or disables recursive file listing.
+ */
public DirectorySampleStream(File[] dirs, FileFilter fileFilter, boolean recursive) {
this.fileFilter = fileFilter;
- isRecursiveScan = recursive;
+ this.recursive = recursive;
List<File> inputDirectoryList = new ArrayList<>(dirs.length);
@@ -64,10 +70,17 @@ public class DirectorySampleStream implements ObjectStream<File> {
directories.addAll(inputDirectories);
}
+ /**
+ * Creates a new directory sample stream.
+ * @param dir The {@link File directory}.
+ * @param fileFilter The {@link FileFilter filter} to apply while enumerating files.
+ * @param recursive Enables or disables recursive file listing.
+ */
public DirectorySampleStream(File dir, FileFilter fileFilter, boolean recursive) {
this(new File[]{dir}, fileFilter, recursive);
}
+ @Override
public File read() throws IOException {
while (textFiles.isEmpty() && !directories.isEmpty()) {
@@ -86,7 +99,7 @@ public class DirectorySampleStream implements ObjectStream<File> {
if (file.isFile()) {
textFiles.push(file);
}
- else if (isRecursiveScan && file.isDirectory()) {
+ else if (recursive && file.isDirectory()) {
directories.push(file);
}
}
@@ -100,6 +113,7 @@ public class DirectorySampleStream implements ObjectStream<File> {
}
}
+ @Override
public void reset() {
directories.clear();
textFiles.clear();
@@ -107,6 +121,14 @@ public class DirectorySampleStream implements ObjectStream<File> {
directories.addAll(inputDirectories);
}
+ /**
+ * {@inheritDoc}
+ * Calling this function has no effect on
+ * the stream.
+ */
+ @Override
public void close() throws IOException {
+
}
+
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/27214015/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java
new file mode 100644
index 0000000..d17188e
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class DirectorySampleStreamTest {
+
+ @Rule
+ public TemporaryFolder tempDirectory = new TemporaryFolder();
+
+ @Test
+ public void directoryTest() throws IOException {
+
+ FileFilter filter = new TempFileNameFilter();
+
+ List<File> files = new ArrayList<>();
+
+ File temp1 = tempDirectory.newFile();
+ files.add(temp1);
+
+ File temp2 = tempDirectory.newFile();
+ files.add(temp2);
+
+ DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, false);
+
+ File file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertNull(file);
+
+ stream.close();
+
+ }
+
+ @Test
+ public void directoryNullFilterTest() throws IOException {
+
+ List<File> files = new ArrayList<>();
+
+ File temp1 = tempDirectory.newFile();
+ files.add(temp1);
+
+ File temp2 = tempDirectory.newFile();
+ files.add(temp2);
+
+ DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), null, false);
+
+ File file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertNull(file);
+
+ stream.close();
+
+ }
+
+ @Test
+ public void recursiveDirectoryTest() throws IOException {
+
+ FileFilter filter = new TempFileNameFilter();
+
+ List<File> files = new ArrayList<>();
+
+ File temp1 = tempDirectory.newFile();
+ files.add(temp1);
+
+ File tempSubDirectory = tempDirectory.newFolder("sub1");
+ File temp2 = File.createTempFile("sub1", ".tmp", tempSubDirectory);
+ files.add(temp2);
+
+ DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, true);
+
+ File file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertNull(file);
+
+ stream.close();
+
+ }
+
+ @Test
+ public void resetDirectoryTest() throws IOException {
+
+ FileFilter filter = new TempFileNameFilter();
+
+ List<File> files = new ArrayList<>();
+
+ File temp1 = tempDirectory.newFile();
+ files.add(temp1);
+
+ File temp2 = tempDirectory.newFile();
+ files.add(temp2);
+
+ DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, false);
+
+ File file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ stream.reset();
+
+ file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertTrue(files.contains(file));
+
+ file = stream.read();
+ Assert.assertNull(file);
+
+ stream.close();
+
+ }
+
+ @Test
+ public void emptyDirectoryTest() throws IOException {
+
+ FileFilter filter = new TempFileNameFilter();
+
+ DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, false);
+
+ Assert.assertNull(stream.read());
+
+ stream.close();
+
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void invalidDirectoryTest() throws IOException {
+
+ FileFilter filter = new TempFileNameFilter();
+
+ DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.newFile(), filter, false);
+
+ Assert.assertNull(stream.read());
+
+ stream.close();
+
+ }
+
+ class TempFileNameFilter implements FileFilter {
+
+ @Override
+ public boolean accept(File file) {
+ return file.isDirectory() || file.getName().endsWith(".tmp");
+ }
+
+ }
+
+}