You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2017/05/04 19:53:49 UTC
[24/50] [abbrv] beam git commit: [BEAM-59] Delete old restrictions on
output file paths
[BEAM-59] Delete old restrictions on output file paths
These predate Apache Beam and are no longer relevant now that Text and Avro are implemented
in the SDK
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/00bee9b7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/00bee9b7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/00bee9b7
Branch: refs/heads/gearpump-runner
Commit: 00bee9b70f6fabbb6bfe0655188f8d2c0a7239f9
Parents: ccbb00e
Author: Dan Halperin <dh...@google.com>
Authored: Mon May 1 23:36:57 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Tue May 2 14:48:21 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/beam/sdk/io/AvroIO.java | 14 --------------
.../main/java/org/apache/beam/sdk/io/TFRecordIO.java | 14 --------------
.../src/main/java/org/apache/beam/sdk/io/TextIO.java | 14 --------------
.../test/java/org/apache/beam/sdk/io/TextIOTest.java | 15 ---------------
4 files changed, 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index 755cdb9..3bb61a2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.io.BaseEncoding;
import java.util.Map;
-import java.util.regex.Pattern;
import javax.annotation.Nullable;
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
@@ -302,7 +301,6 @@ public class AvroIO {
* in a common extension, if given by {@link #withSuffix}.
*/
public Write<T> to(String filenamePrefix) {
- validateOutputComponent(filenamePrefix);
return toBuilder().setFilenamePrefix(filenamePrefix).build();
}
@@ -317,7 +315,6 @@ public class AvroIO {
* <p>See {@link ShardNameTemplate} for a description of shard templates.
*/
public Write<T> withSuffix(String filenameSuffix) {
- validateOutputComponent(filenameSuffix);
return toBuilder().setFilenameSuffix(filenameSuffix).build();
}
@@ -474,17 +471,6 @@ public class AvroIO {
}
}
- // Pattern which matches old-style shard output patterns, which are now
- // disallowed.
- private static final Pattern SHARD_OUTPUT_PATTERN = Pattern.compile("@([0-9]+|\\*)");
-
- private static void validateOutputComponent(String partialFilePattern) {
- checkArgument(
- !SHARD_OUTPUT_PATTERN.matcher(partialFilePattern).find(),
- "Output name components are not allowed to contain @* or @N patterns: "
- + partialFilePattern);
- }
-
/////////////////////////////////////////////////////////////////////////////
/** Disallow construction of utility class. */
http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
index 8a1870e..fe0b97d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TFRecordIO.java
@@ -30,7 +30,6 @@ import java.nio.ByteOrder;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.NoSuchElementException;
-import java.util.regex.Pattern;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.ByteArrayCoder;
import org.apache.beam.sdk.coders.Coder;
@@ -268,7 +267,6 @@ public class TFRecordIO {
* in a common extension, if given by {@link #withSuffix(String)}.
*/
public Write to(String filenamePrefix) {
- validateOutputComponent(filenamePrefix);
return to(StaticValueProvider.of(filenamePrefix));
}
@@ -285,7 +283,6 @@ public class TFRecordIO {
* @see ShardNameTemplate
*/
public Write withSuffix(String nameExtension) {
- validateOutputComponent(nameExtension);
return toBuilder().setFilenameSuffix(nameExtension).build();
}
@@ -422,17 +419,6 @@ public class TFRecordIO {
}
}
- // Pattern which matches old-style shard output patterns, which are now
- // disallowed.
- private static final Pattern SHARD_OUTPUT_PATTERN = Pattern.compile("@([0-9]+|\\*)");
-
- private static void validateOutputComponent(String partialFilePattern) {
- checkArgument(
- !SHARD_OUTPUT_PATTERN.matcher(partialFilePattern).find(),
- "Output name components are not allowed to contain @* or @N patterns: "
- + partialFilePattern);
- }
-
//////////////////////////////////////////////////////////////////////////////
/** Disable construction of utility class. */
http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
index 1f9b7a0..6b08e1f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
@@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.auto.value.AutoValue;
-import java.util.regex.Pattern;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.StringUtf8Coder;
@@ -290,7 +289,6 @@ public class TextIO {
* in a common extension, if given by {@link #withSuffix(String)}.
*/
public Write to(String filenamePrefix) {
- validateOutputComponent(filenamePrefix);
return to(StaticValueProvider.of(filenamePrefix));
}
@@ -310,7 +308,6 @@ public class TextIO {
* @see ShardNameTemplate
*/
public Write withSuffix(String nameExtension) {
- validateOutputComponent(nameExtension);
return toBuilder().setFilenameSuffix(nameExtension).build();
}
@@ -505,17 +502,6 @@ public class TextIO {
}
}
- // Pattern which matches old-style shard output patterns, which are now
- // disallowed.
- private static final Pattern SHARD_OUTPUT_PATTERN = Pattern.compile("@([0-9]+|\\*)");
-
- private static void validateOutputComponent(String partialFilePattern) {
- checkArgument(
- !SHARD_OUTPUT_PATTERN.matcher(partialFilePattern).find(),
- "Output name components are not allowed to contain @* or @N patterns: "
- + partialFilePattern);
- }
-
//////////////////////////////////////////////////////////////////////////////
/** Disable construction of utility class. */
http://git-wip-us.apache.org/repos/asf/beam/blob/00bee9b7/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
index f30b52f..66b605f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
@@ -540,21 +540,6 @@ public class TextIOTest {
displayData, hasItem(hasDisplayItem(hasValue(startsWith(outputPath)))));
}
- @Test
- public void testUnsupportedFilePattern() throws IOException {
- p.enableAbandonedNodeEnforcement(false);
- // Windows doesn't like resolving paths with * in them.
- String filename = tempFolder.resolve("output@5").toString();
-
- PCollection<String> input =
- p.apply(Create.of(Arrays.asList(LINES_ARRAY))
- .withCoder(StringUtf8Coder.of()));
-
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Output name components are not allowed to contain");
- input.apply(TextIO.write().to(filename));
- }
-
/** Options for testing. */
public interface RuntimeTestOptions extends PipelineOptions {
ValueProvider<String> getInput();