You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/01/29 16:21:58 UTC
[12/50] beam git commit: [BEAM-59] Beam FileSystem.setDefaultConfig:
remove scheme from the signature.
[BEAM-59] Beam FileSystem.setDefaultConfig: remove scheme from the signature.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4cdd8771
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4cdd8771
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4cdd8771
Branch: refs/heads/python-sdk
Commit: 4cdd87718c3d0719b7c0e421b9cbaf4eb902672e
Parents: 1148be6
Author: Pei He <pe...@google.com>
Authored: Mon Jan 23 18:08:44 2017 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Jan 24 15:54:53 2017 -0800
----------------------------------------------------------------------
.../org/apache/beam/sdk/io/FileSystems.java | 32 +++++++------------
.../org/apache/beam/sdk/io/FileSystemsTest.java | 33 +++-----------------
2 files changed, 15 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4cdd8771/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index d086ec6..e19c1e4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.sdk.io;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
@@ -53,6 +53,8 @@ public class FileSystems {
private static final Map<String, FileSystemRegistrar> SCHEME_TO_REGISTRAR =
new ConcurrentHashMap<>();
+ private static PipelineOptions defaultConfig;
+
private static final Map<String, PipelineOptions> SCHEME_TO_DEFAULT_CONFIG =
new ConcurrentHashMap<>();
@@ -78,27 +80,12 @@ public class FileSystems {
}
/**
- * Sets the default configuration to be used with a {@link FileSystemRegistrar} for the provided
- * {@code scheme}.
+ * Sets the default configuration in workers.
*
- * <p>Syntax: <pre>scheme = alpha *( alpha | digit | "+" | "-" | "." )</pre>
- * Upper case letters are treated as the same as lower case letters.
+ * <p>It will be used in {@link FileSystemRegistrar FileSystemRegistrars} for all schemes.
*/
- public static void setDefaultConfig(String scheme, PipelineOptions options) {
- String lowerCaseScheme = checkNotNull(scheme, "scheme").toLowerCase();
- checkArgument(
- URI_SCHEME_PATTERN.matcher(lowerCaseScheme).matches(),
- String.format("Scheme: [%s] doesn't match URI syntax: %s",
- lowerCaseScheme, URI_SCHEME_PATTERN.pattern()));
- checkArgument(
- SCHEME_TO_REGISTRAR.containsKey(lowerCaseScheme),
- String.format("No FileSystemRegistrar found for scheme: [%s].", lowerCaseScheme));
- SCHEME_TO_DEFAULT_CONFIG.put(lowerCaseScheme, checkNotNull(options, "options"));
- }
-
- @VisibleForTesting
- static PipelineOptions getDefaultConfig(String scheme) {
- return SCHEME_TO_DEFAULT_CONFIG.get(scheme.toLowerCase());
+ public static void setDefaultConfigInWorkers(PipelineOptions options) {
+ defaultConfig = checkNotNull(options, "options");
}
/**
@@ -106,9 +93,12 @@ public class FileSystems {
*/
@VisibleForTesting
static FileSystem getFileSystemInternal(URI uri) {
+ checkState(
+ defaultConfig != null,
+ "Expect the runner have called setDefaultConfigInWorkers().");
String lowerCaseScheme = (uri.getScheme() != null
? uri.getScheme().toLowerCase() : LocalFileSystemRegistrar.LOCAL_FILE_SCHEME);
- return getRegistrarInternal(lowerCaseScheme).fromOptions(getDefaultConfig(lowerCaseScheme));
+ return getRegistrarInternal(lowerCaseScheme).fromOptions(defaultConfig);
}
/**
http://git-wip-us.apache.org/repos/asf/beam/blob/4cdd8771/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
index 9b41b98..113a562 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
@@ -17,8 +17,6 @@
*/
package org.apache.beam.sdk.io;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import com.google.common.collect.Sets;
@@ -26,6 +24,7 @@ import java.net.URI;
import javax.annotation.Nullable;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -41,33 +40,9 @@ public class FileSystemsTest {
@Rule
public ExpectedException thrown = ExpectedException.none();
- @Test
- public void testSetDefaultConfig() throws Exception {
- PipelineOptions first = PipelineOptionsFactory.create();
- PipelineOptions second = PipelineOptionsFactory.create();
- FileSystems.setDefaultConfig("file", first);
- assertEquals(first, FileSystems.getDefaultConfig("file"));
- assertEquals(first, FileSystems.getDefaultConfig("FILE"));
-
- FileSystems.setDefaultConfig("FILE", second);
- assertNotEquals(first, FileSystems.getDefaultConfig("file"));
- assertNotEquals(first, FileSystems.getDefaultConfig("FILE"));
- assertEquals(second, FileSystems.getDefaultConfig("file"));
- assertEquals(second, FileSystems.getDefaultConfig("FILE"));
- }
-
- @Test
- public void testSetDefaultConfigNotFound() throws Exception {
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("No FileSystemRegistrar found for scheme: [gs-s3].");
- FileSystems.setDefaultConfig("gs-s3", PipelineOptionsFactory.create());
- }
-
- @Test
- public void testSetDefaultConfigInvalidScheme() throws Exception {
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("Scheme: [gs:] doesn't match URI syntax");
- FileSystems.setDefaultConfig("gs:", PipelineOptionsFactory.create());
+ @Before
+ public void setup() {
+ FileSystems.setDefaultConfigInWorkers(PipelineOptionsFactory.create());
}
@Test