You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2017/08/28 16:02:13 UTC
incubator-gobblin git commit: [GOBBLIN-56] Changing statestore
checker to work with any state store Making non-required the state store url
Adding required text to help for required properties Fixing issue where it
read second paramater as value due to u
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 841193a37 -> 4e9453fba
[GOBBLIN-56] Changing statestore checker to work with any state store
Making non-required the state store url
Adding required text to help for required properties
Fixing issue where it read second paramater as value due to using hasArgs instead of hasArg
Closes #2077 from
treff7es/mysql_state_store_checker
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/4e9453fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/4e9453fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/4e9453fb
Branch: refs/heads/master
Commit: 4e9453fba6e1ceb51077ec1fe5871f4d65109ff1
Parents: 841193a
Author: treff7es <ta...@prezi.com>
Authored: Mon Aug 28 09:01:13 2017 -0700
Committer: Hung Tran <hu...@linkedin.com>
Committed: Mon Aug 28 09:01:13 2017 -0700
----------------------------------------------------------------------
.../runtime/util/JobStateToJsonConverter.java | 34 +++++++++++-------
.../util/JobStateToJsonConverterTest.java | 38 +++++++++++++++++---
2 files changed, 55 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4e9453fb/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
----------------------------------------------------------------------
diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
index 5860daa..71a66db 100644
--- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
+++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
@@ -33,9 +33,8 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,9 +42,10 @@ import com.google.common.io.Closer;
import com.google.gson.stream.JsonWriter;
import org.apache.gobblin.configuration.ConfigurationKeys;
+import org.apache.gobblin.metastore.DatasetStateStore;
import org.apache.gobblin.metastore.StateStore;
-import org.apache.gobblin.runtime.FsDatasetStateStore;
import org.apache.gobblin.runtime.JobState;
+import org.apache.gobblin.util.ConfigUtils;
import org.apache.gobblin.util.JobConfigurationUtils;
@@ -66,11 +66,13 @@ public class JobStateToJsonConverter {
public JobStateToJsonConverter(Properties props, String storeUrl, boolean keepConfig) throws IOException {
Configuration conf = new Configuration();
JobConfigurationUtils.putPropertiesIntoConfiguration(props, conf);
- Path storePath = new Path(storeUrl);
- FileSystem fs = storePath.getFileSystem(conf);
- String storeRootDir = storePath.toUri().getPath();
- this.jobStateStore = new FsDatasetStateStore(fs, storeRootDir);
+
+ if (StringUtils.isNotBlank(storeUrl)) {
+ props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeUrl);
+ }
+
this.keepConfig = keepConfig;
+ this.jobStateStore = (StateStore) DatasetStateStore.buildDatasetStateStore(ConfigUtils.propertiesToConfig(props));
}
/**
@@ -154,19 +156,19 @@ public class JobStateToJsonConverter {
@SuppressWarnings("all")
public static void main(String[] args) throws Exception {
Option sysConfigOption = Option.builder("sc").argName("system configuration file")
- .desc("Gobblin system configuration file").longOpt("sysconfig").hasArgs().build();
+ .desc("Gobblin system configuration file (required if no state store URL specified)").longOpt("sysconfig").hasArg().build();
Option storeUrlOption = Option.builder("u").argName("gobblin state store URL")
- .desc("Gobblin state store root path URL").longOpt("storeurl").hasArgs().required().build();
- Option jobNameOption = Option.builder("n").argName("gobblin job name").desc("Gobblin job name").longOpt("name")
- .hasArgs().required().build();
+ .desc("Gobblin state store root path URL (required if no sysconfig specified)").longOpt("storeurl").hasArg().build();
+ Option jobNameOption = Option.builder("n").argName("gobblin job name").desc("Gobblin job name (required)").longOpt("name")
+ .hasArg().required().build();
Option jobIdOption =
- Option.builder("i").argName("gobblin job id").desc("Gobblin job id").longOpt("id").hasArgs().build();
+ Option.builder("i").argName("gobblin job id").desc("Gobblin job id").longOpt("id").hasArg().build();
Option convertAllOption =
Option.builder("a").desc("Whether to convert all past job states of the given job").longOpt("all").build();
Option keepConfigOption =
Option.builder("kc").desc("Whether to keep all configuration properties").longOpt("keepConfig").build();
Option outputToFile =
- Option.builder("t").argName("output file name").desc("Output file name").longOpt("toFile").hasArgs().build();
+ Option.builder("t").argName("output file name").desc("Output file name").longOpt("toFile").hasArg().build();
Options options = new Options();
options.addOption(sysConfigOption);
@@ -187,6 +189,12 @@ public class JobStateToJsonConverter {
System.exit(1);
}
+ if (!cmd.hasOption(sysConfigOption.getLongOpt()) && !cmd.hasOption(storeUrlOption.getLongOpt()) ){
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("JobStateToJsonConverter", options);
+ System.exit(1);
+ }
+
Properties sysConfig = new Properties();
if (cmd.hasOption(sysConfigOption.getLongOpt())) {
sysConfig = JobConfigurationUtils.fileToProperties(cmd.getOptionValue(sysConfigOption.getLongOpt()));
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4e9453fb/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
----------------------------------------------------------------------
diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
index e4bf4f9..66ca54f 100644
--- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
+++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java
@@ -16,17 +16,22 @@
*/
package org.apache.gobblin.runtime.util;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.stream.JsonReader;
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.Properties;
+
import org.testng.Assert;
import org.testng.annotations.Test;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.stream.JsonReader;
+
+import org.apache.gobblin.configuration.ConfigurationKeys;
+
+
@Test(groups = { "gobblin.runtime" })
public class JobStateToJsonConverterTest {
@@ -40,7 +45,9 @@ public class JobStateToJsonConverterTest {
public void testJsonKeepConfig()
throws IOException {
String stateStorePath = getClass().getClassLoader().getResource(TEST_STORE).getPath();
+
boolean keepConfig = true;
+
JobStateToJsonConverter converter = new JobStateToJsonConverter(new Properties(), stateStorePath, keepConfig);
StringWriter stringWriter = new StringWriter();
@@ -53,4 +60,27 @@ public class JobStateToJsonConverterTest {
Assert.assertNotNull(taskState.getAsJsonObject().get(PROPERTIES));
}
}
+
+ @Test
+ public void testJsonKeepConfigWithoutStoreUrl()
+ throws IOException {
+ String stateStorePath = getClass().getClassLoader().getResource(TEST_STORE).getPath();
+ Properties properties = new Properties();
+
+ properties.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStorePath);
+
+ boolean keepConfig = true;
+
+ JobStateToJsonConverter converter = new JobStateToJsonConverter(properties, null, keepConfig);
+
+ StringWriter stringWriter = new StringWriter();
+ converter.convert(TEST_JOB, stringWriter);
+
+ JsonObject json = new JsonParser().parse(new JsonReader(new StringReader(stringWriter.toString()))).getAsJsonObject();
+
+ Assert.assertNotNull(json.get(PROPERTIES));
+ for (JsonElement taskState: json.get(TASK_STATES).getAsJsonArray()) {
+ Assert.assertNotNull(taskState.getAsJsonObject().get(PROPERTIES));
+ }
+ }
}