You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2018/10/09 14:54:04 UTC

[flink] branch master updated (d376f1e -> a363146)

This is an automated email from the ASF dual-hosted git repository.

gary pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from d376f1e  [FLINK-10465][tests] Do not fail if /etc/service does not exists.
     new 26d9a7e  [FLINK-10399] Refactor ParameterTool#fromArgs
     new 959eb7d  [FLINK-10399] Use String#isEmpty() to check for empty string.
     new a363146  [FLINK-10399] Rely on exception propgation to fail tests.

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../apache/flink/api/java/utils/ParameterTool.java | 108 +++++++--------------
 .../api/java/utils/AbstractParameterToolTest.java  |  29 +++---
 .../flink/api/java/utils/ParameterToolTest.java    |  19 +++-
 3 files changed, 61 insertions(+), 95 deletions(-)


[flink] 02/03: [FLINK-10399] Use String#isEmpty() to check for empty string.

Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 959eb7dd2f0b91f40343694e9a43aafde7a86b0f
Author: gyao <ga...@data-artisans.com>
AuthorDate: Tue Oct 9 14:54:13 2018 +0200

    [FLINK-10399] Use String#isEmpty() to check for empty string.
---
 .../src/main/java/org/apache/flink/api/java/utils/ParameterTool.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java b/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java
index 9adbcaf..33b6ab9 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java
@@ -84,7 +84,7 @@ public class ParameterTool extends ExecutionConfig.GlobalJobParameters implement
 						Arrays.toString(args), args[i]));
 			}
 
-			if (key.length() == 0) {
+			if (key.isEmpty()) {
 				throw new IllegalArgumentException(
 					"The input " + Arrays.toString(args) + " contains an empty argument");
 			}


[flink] 03/03: [FLINK-10399] Rely on exception propgation to fail tests.

Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit a363146453c0d202f06c3c0b9f06b804e17b7e8b
Author: gyao <ga...@data-artisans.com>
AuthorDate: Tue Oct 9 14:55:44 2018 +0200

    [FLINK-10399] Rely on exception propgation to fail tests.
---
 .../apache/flink/api/java/utils/AbstractParameterToolTest.java    | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java b/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java
index a5d2f87..40cf28e 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java
@@ -47,9 +47,8 @@ public abstract class AbstractParameterToolTest {
 			byte[] b = InstantiationUtil.serializeObject(parameter);
 			final ParameterTool copy = InstantiationUtil.deserializeObject(b, getClass().getClassLoader());
 			internalValidate(copy);
-		} catch (Exception e) {
-			e.printStackTrace();
-			Assert.fail(e.getMessage());
+		} catch (IOException | ClassNotFoundException e) {
+			throw new RuntimeException(e);
 		}
 	}
 
@@ -82,8 +81,7 @@ public abstract class AbstractParameterToolTest {
 			Assert.assertTrue(defaultProps.containsKey("input"));
 
 		} catch (IOException e) {
-			e.printStackTrace();
-			Assert.fail(e.getMessage());
+			throw new RuntimeException(e);
 		}
 	}
 }


[flink] 01/03: [FLINK-10399] Refactor ParameterTool#fromArgs

Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 26d9a7eb8cd01b9e1105b7b3a5a2909c3c1ad630
Author: tison <wa...@gmail.com>
AuthorDate: Sun Sep 23 15:50:57 2018 +0800

    [FLINK-10399] Refactor ParameterTool#fromArgs
    
    This closes #6737.
---
 .../apache/flink/api/java/utils/ParameterTool.java | 108 +++++++--------------
 .../api/java/utils/AbstractParameterToolTest.java  |  27 +++---
 .../flink/api/java/utils/ParameterToolTest.java    |  19 +++-
 3 files changed, 61 insertions(+), 93 deletions(-)

diff --git a/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java b/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java
index 7518fa0..9adbcaf 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/utils/ParameterTool.java
@@ -68,81 +68,43 @@ public class ParameterTool extends ExecutionConfig.GlobalJobParameters implement
 	 * @return A {@link ParameterTool}
 	 */
 	public static ParameterTool fromArgs(String[] args) {
-		Map<String, String> map = new HashMap<String, String>(args.length / 2);
-
-		String key = null;
-		String value = null;
-		boolean expectValue = false;
-		for (String arg : args) {
-			// check for -- argument
-			if (arg.startsWith("--")) {
-				if (expectValue) {
-					// we got into a new key, even though we were a value --> current key is one without value
-					if (value != null) {
-						throw new IllegalStateException("Unexpected state");
-					}
-					map.put(key, NO_VALUE_KEY);
-					// key will be overwritten in the next step
-				}
-				key = arg.substring(2);
-				expectValue = true;
-			} // check for - argument
-			else if (arg.startsWith("-")) {
-				// we are waiting for a value, so this is a - prefixed value (negative number)
-				if (expectValue) {
-
-					if (NumberUtils.isNumber(arg)) {
-						// negative number
-						value = arg;
-						expectValue = false;
-					} else {
-						if (value != null) {
-							throw new IllegalStateException("Unexpected state");
-						}
-						// We waited for a value but found a new key. So the previous key doesnt have a value.
-						map.put(key, NO_VALUE_KEY);
-						key = arg.substring(1);
-						expectValue = true;
-					}
-				} else {
-					// we are not waiting for a value, so its an argument
-					key = arg.substring(1);
-					expectValue = true;
-				}
+		final Map<String, String> map = new HashMap<>(args.length / 2);
+
+		int i = 0;
+		while (i < args.length) {
+			final String key;
+
+			if (args[i].startsWith("--")) {
+				key = args[i].substring(2);
+			} else if (args[i].startsWith("-")) {
+				key = args[i].substring(1);
 			} else {
-				if (expectValue) {
-					value = arg;
-					expectValue = false;
-				} else {
-					throw new RuntimeException("Error parsing arguments '" + Arrays.toString(args) + "' on '" + arg + "'. Unexpected value. Please prefix values with -- or -.");
-				}
+				throw new IllegalArgumentException(
+					String.format("Error parsing arguments '%s' on '%s'. Please prefix keys with -- or -.",
+						Arrays.toString(args), args[i]));
 			}
 
-			if (value == null && key == null) {
-				throw new IllegalStateException("Value and key can not be null at the same time");
-			}
-			if (key != null && value == null && !expectValue) {
-				throw new IllegalStateException("Value expected but flag not set");
-			}
-			if (key != null && value != null) {
-				map.put(key, value);
-				key = null;
-				value = null;
-				expectValue = false;
-			}
-			if (key != null && key.length() == 0) {
-				throw new IllegalArgumentException("The input " + Arrays.toString(args) + " contains an empty argument");
+			if (key.length() == 0) {
+				throw new IllegalArgumentException(
+					"The input " + Arrays.toString(args) + " contains an empty argument");
 			}
 
-			if (key != null && !expectValue) {
+			i += 1; // try to find the value
+
+			if (i >= args.length) {
+				map.put(key, NO_VALUE_KEY);
+			} else if (NumberUtils.isNumber(args[i])) {
+				map.put(key, args[i]);
+				i += 1;
+			} else if (args[i].startsWith("--") || args[i].startsWith("-")) {
+				// the argument cannot be a negative number because we checked earlier
+				// -> the next argument is a parameter name
 				map.put(key, NO_VALUE_KEY);
-				key = null;
-				expectValue = false;
+			} else {
+				map.put(key, args[i]);
+				i += 1;
 			}
 		}
-		if (key != null) {
-			map.put(key, NO_VALUE_KEY);
-		}
 
 		return fromMap(map);
 	}
@@ -497,7 +459,7 @@ public class ParameterTool extends ExecutionConfig.GlobalJobParameters implement
 	// --------------- Internals
 
 	protected void addToDefaults(String key, String value) {
-		String currentValue = defaultData.get(key);
+		final String currentValue = defaultData.get(key);
 		if (currentValue == null) {
 			if (value == null) {
 				value = DEFAULT_UNDEFINED;
@@ -520,7 +482,7 @@ public class ParameterTool extends ExecutionConfig.GlobalJobParameters implement
 	 * @return A {@link Configuration}
 	 */
 	public Configuration getConfiguration() {
-		Configuration conf = new Configuration();
+		final Configuration conf = new Configuration();
 		for (Map.Entry<String, String> entry : data.entrySet()) {
 			conf.setString(entry.getKey(), entry.getValue());
 		}
@@ -559,7 +521,7 @@ public class ParameterTool extends ExecutionConfig.GlobalJobParameters implement
 	 * @throws IOException If overwrite is not allowed and the file exists
 	 */
 	public void createPropertiesFile(String pathToFile, boolean overwrite) throws IOException {
-		File file = new File(pathToFile);
+		final File file = new File(pathToFile);
 		if (file.exists()) {
 			if (overwrite) {
 				file.delete();
@@ -567,7 +529,7 @@ public class ParameterTool extends ExecutionConfig.GlobalJobParameters implement
 				throw new RuntimeException("File " + pathToFile + " exists and overwriting is not allowed");
 			}
 		}
-		Properties defaultProps = new Properties();
+		final Properties defaultProps = new Properties();
 		defaultProps.putAll(this.defaultData);
 		try (final OutputStream out = new FileOutputStream(file)) {
 			defaultProps.store(out, "Default file created by Flink's ParameterUtil.createPropertiesFile()");
@@ -588,11 +550,11 @@ public class ParameterTool extends ExecutionConfig.GlobalJobParameters implement
 	 * @return The Merged {@link ParameterTool}
 	 */
 	public ParameterTool mergeWith(ParameterTool other) {
-		Map<String, String> resultData = new HashMap<>(data.size() + other.data.size());
+		final Map<String, String> resultData = new HashMap<>(data.size() + other.data.size());
 		resultData.putAll(data);
 		resultData.putAll(other.data);
 
-		ParameterTool ret = new ParameterTool(resultData);
+		final ParameterTool ret = new ParameterTool(resultData);
 
 		final HashSet<String> requestedParametersLeft = new HashSet<>(data.keySet());
 		requestedParametersLeft.removeAll(unrequestedParameters);
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java b/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java
index 784a50e..a5d2f87 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/utils/AbstractParameterToolTest.java
@@ -30,8 +30,6 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.util.Properties;
 
-import static org.junit.Assert.fail;
-
 /**
  * Base class for tests for {@link ParameterTool}.
  */
@@ -42,40 +40,39 @@ public abstract class AbstractParameterToolTest {
 
 	protected void validate(ParameterTool parameter) {
 		ClosureCleaner.ensureSerializable(parameter);
-		validatePrivate(parameter);
+		internalValidate(parameter);
 
 		// -------- test behaviour after serialization ------------
-		ParameterTool copy = null;
 		try {
 			byte[] b = InstantiationUtil.serializeObject(parameter);
-			copy = InstantiationUtil.deserializeObject(b, getClass().getClassLoader());
+			final ParameterTool copy = InstantiationUtil.deserializeObject(b, getClass().getClassLoader());
+			internalValidate(copy);
 		} catch (Exception e) {
-			fail();
+			e.printStackTrace();
+			Assert.fail(e.getMessage());
 		}
-		validatePrivate(copy);
 	}
 
-	private void validatePrivate(ParameterTool parameter) {
+	private void internalValidate(ParameterTool parameter) {
 		Assert.assertEquals("myInput", parameter.getRequired("input"));
 		Assert.assertEquals("myDefaultValue", parameter.get("output", "myDefaultValue"));
-		Assert.assertEquals(null, parameter.get("whatever"));
+		Assert.assertNull(parameter.get("whatever"));
 		Assert.assertEquals(15L, parameter.getLong("expectedCount", -1L));
 		Assert.assertTrue(parameter.getBoolean("thisIsUseful", true));
 		Assert.assertEquals(42, parameter.getByte("myDefaultByte", (byte) 42));
 		Assert.assertEquals(42, parameter.getShort("myDefaultShort", (short) 42));
 
-		Configuration config = parameter.getConfiguration();
+		final Configuration config = parameter.getConfiguration();
 		Assert.assertEquals(15L, config.getLong("expectedCount", -1L));
 
-		Properties props = parameter.getProperties();
+		final Properties props = parameter.getProperties();
 		Assert.assertEquals("myInput", props.getProperty("input"));
-		props = null;
 
 		// -------- test the default file creation ------------
 		try {
-			String pathToFile = tmp.newFile().getAbsolutePath();
+			final String pathToFile = tmp.newFile().getAbsolutePath();
 			parameter.createPropertiesFile(pathToFile);
-			Properties defaultProps = new Properties();
+			final Properties defaultProps = new Properties();
 			try (FileInputStream fis = new FileInputStream(pathToFile)) {
 				defaultProps.load(fis);
 			}
@@ -85,8 +82,8 @@ public abstract class AbstractParameterToolTest {
 			Assert.assertTrue(defaultProps.containsKey("input"));
 
 		} catch (IOException e) {
-			Assert.fail(e.getMessage());
 			e.printStackTrace();
+			Assert.fail(e.getMessage());
 		}
 	}
 }
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/utils/ParameterToolTest.java b/flink-java/src/test/java/org/apache/flink/api/java/utils/ParameterToolTest.java
index ccd472b..69f95de 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/utils/ParameterToolTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/utils/ParameterToolTest.java
@@ -57,9 +57,12 @@ public class ParameterToolTest extends AbstractParameterToolTest {
 
 	// ----- Parser tests -----------------
 
-	@Test(expected = RuntimeException.class)
-	public void testIllegalArgs() {
-		ParameterTool.fromArgs(new String[]{"berlin"});
+	@Test
+	public void testThrowExceptionIfParameterIsNotPrefixed() {
+		exception.expect(IllegalArgumentException.class);
+		exception.expectMessage("Error parsing arguments '[a]' on 'a'. Please prefix keys with -- or -.");
+
+		ParameterTool.fromArgs(new String[]{"a"});
 	}
 
 	@Test
@@ -100,13 +103,19 @@ public class ParameterToolTest extends AbstractParameterToolTest {
 		Assert.assertTrue(parameter.has("f"));
 	}
 
-	@Test(expected = IllegalArgumentException.class)
+	@Test
 	public void testEmptyVal() {
+		exception.expect(IllegalArgumentException.class);
+		exception.expectMessage("The input [--a, -b, --] contains an empty argument");
+
 		ParameterTool.fromArgs(new String[]{"--a", "-b", "--"});
 	}
 
-	@Test(expected = IllegalArgumentException.class)
+	@Test
 	public void testEmptyValShort() {
+		exception.expect(IllegalArgumentException.class);
+		exception.expectMessage("The input [--a, -b, -] contains an empty argument");
+
 		ParameterTool.fromArgs(new String[]{"--a", "-b", "-"});
 	}