You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/06/20 23:21:58 UTC
[1/2] incubator-beam git commit: [BEAM-363] Return false for non-AUTO
compression mode
Repository: incubator-beam
Updated Branches:
refs/heads/master 682a19c4e -> 2b9906e8d
[BEAM-363] Return false for non-AUTO compression mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/278a76e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/278a76e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/278a76e0
Branch: refs/heads/master
Commit: 278a76e096ced0ba51b5ed0ae8205bbd32491c77
Parents: 682a19c
Author: Dan Halperin <dh...@google.com>
Authored: Mon Jun 20 15:44:54 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Jun 20 15:44:54 2016 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/io/CompressedSource.java | 2 +-
.../beam/sdk/io/CompressedSourceTest.java | 50 ++++++++++++++++++++
2 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/278a76e0/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
index 8bccf5f..a5c54b3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
@@ -292,7 +292,7 @@ public class CompressedSource<T> extends FileBasedSource<T> {
(FileNameBasedDecompressingChannelFactory) channelFactory;
return !fileNameBasedChannelFactory.isCompressed(getFileOrPatternSpec());
}
- return true;
+ return false;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/278a76e0/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
index c34d2dd..abf1de3 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
@@ -97,6 +97,56 @@ public class CompressedSourceTest {
}
/**
+ * Test splittability of files in AUTO mode.
+ */
+ @Test
+ public void testAutoSplittable() throws Exception {
+ CompressedSource<Byte> source;
+
+ // GZip files are not splittable
+ source = CompressedSource.from(new ByteSource("input.gz", 1));
+ assertFalse(source.isSplittable());
+ source = CompressedSource.from(new ByteSource("input.GZ", 1));
+ assertFalse(source.isSplittable());
+
+ // BZ2 files are not splittable
+ source = CompressedSource.from(new ByteSource("input.bz2", 1));
+ assertFalse(source.isSplittable());
+ source = CompressedSource.from(new ByteSource("input.BZ2", 1));
+ assertFalse(source.isSplittable());
+
+ // Other extensions are assumed to be splittable.
+ source = CompressedSource.from(new ByteSource("input.txt", 1));
+ assertTrue(source.isSplittable());
+ source = CompressedSource.from(new ByteSource("input.csv", 1));
+ assertTrue(source.isSplittable());
+ }
+
+ /**
+ * Test splittability of files in GZIP mode -- none should be splittable.
+ */
+ @Test
+ public void testGzipSplittable() throws Exception {
+ CompressedSource<Byte> source;
+
+ // GZip files are not splittable
+ source = CompressedSource.from(new ByteSource("input.gz", 1))
+ .withDecompression(CompressionMode.GZIP);
+ assertFalse(source.isSplittable());
+ source = CompressedSource.from(new ByteSource("input.GZ", 1))
+ .withDecompression(CompressionMode.GZIP);
+ assertFalse(source.isSplittable());
+
+ // Other extensions are also not splittable.
+ source = CompressedSource.from(new ByteSource("input.txt", 1))
+ .withDecompression(CompressionMode.GZIP);
+ assertFalse(source.isSplittable());
+ source = CompressedSource.from(new ByteSource("input.csv", 1))
+ .withDecompression(CompressionMode.GZIP);
+ assertFalse(source.isSplittable());
+ }
+
+ /**
* Test reading nonempty input with bzip2.
*/
@Test
[2/2] incubator-beam git commit: Closes #504
Posted by dh...@apache.org.
Closes #504
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2b9906e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2b9906e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2b9906e8
Branch: refs/heads/master
Commit: 2b9906e8d97c254706656ac68eec85f29f946627
Parents: 682a19c 278a76e
Author: Dan Halperin <dh...@google.com>
Authored: Mon Jun 20 16:21:45 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Jun 20 16:21:45 2016 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/io/CompressedSource.java | 2 +-
.../beam/sdk/io/CompressedSourceTest.java | 50 ++++++++++++++++++++
2 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------