[gobblin] branch master updated: [GOBBLIN-1462] Ensure FsSpecConsumer handles config keys which are prefixes of other keys

[su...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 1bc7e82  [GOBBLIN-1462] Ensure FsSpecConsumer handles config keys which are prefixes of other keys
1bc7e82 is described below

commit 1bc7e822ea27bdf90cc2f3fe895d2d6dc72e3f53
Author: suvasude <su...@linkedin.biz>
AuthorDate: Thu Jun 10 13:04:08 2021 -0700

    [GOBBLIN-1462] Ensure FsSpecConsumer handles config keys which are prefixes of other keys
    
    Closes #3302 from sv2000/typesafeConfigPrefix
---
 .../apache/gobblin/runtime/api/FsSpecConsumer.java |  8 ++++++--
 .../gobblin/runtime/api/FsSpecProducerTest.java    | 22 ++++++++++++++++++----
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/FsSpecConsumer.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/FsSpecConsumer.java
index ccc0a49..7e3d286 100644
--- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/FsSpecConsumer.java
+++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/FsSpecConsumer.java
@@ -45,6 +45,7 @@ import lombok.extern.slf4j.Slf4j;
 
 import org.apache.gobblin.runtime.job_spec.AvroJobSpec;
 import org.apache.gobblin.util.CompletedFuture;
+import org.apache.gobblin.util.ConfigUtils;
 import org.apache.gobblin.util.filters.HiddenFilter;
 
 
@@ -112,8 +113,11 @@ public class FsSpecConsumer implements SpecConsumer<Spec> {
         JobSpec.Builder jobSpecBuilder = new JobSpec.Builder(avroJobSpec.getUri());
         Properties props = new Properties();
         props.putAll(avroJobSpec.getProperties());
-        jobSpecBuilder.withJobCatalogURI(avroJobSpec.getUri()).withVersion(avroJobSpec.getVersion())
-            .withDescription(avroJobSpec.getDescription()).withConfigAsProperties(props);
+        jobSpecBuilder.withJobCatalogURI(avroJobSpec.getUri())
+            .withVersion(avroJobSpec.getVersion())
+            .withDescription(avroJobSpec.getDescription())
+            .withConfigAsProperties(props)
+            .withConfig(ConfigUtils.propertiesToConfig(props));
 
         try {
           if (!avroJobSpec.getTemplateUri().isEmpty()) {
diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/FsSpecProducerTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/FsSpecProducerTest.java
index 15d07be..52dcf8b 100644
--- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/FsSpecProducerTest.java
+++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/FsSpecProducerTest.java
@@ -34,6 +34,8 @@ import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigValueFactory;
 
+import org.apache.gobblin.util.ConfigUtils;
+
 
 public class FsSpecProducerTest {
   private FsSpecProducer _fsSpecProducer;
@@ -50,10 +52,18 @@ public class FsSpecProducerTest {
   }
 
   private JobSpec createTestJobSpec() throws URISyntaxException {
-    JobSpec jobSpec = JobSpec.builder("testJob").withConfig(ConfigFactory.empty().
-        withValue("key1", ConfigValueFactory.fromAnyRef("val1")).
-        withValue("key2", ConfigValueFactory.fromAnyRef("val2"))).
-        withVersion("1").withDescription("").withTemplate(new URI("FS:///")).build();
+    Properties properties = new Properties();
+    properties.put("key1", "val1");
+    properties.put("key2", "val2");
+    //Introduce a key which is a prefix of another key and ensure it is correctly handled in the code
+    properties.put("key3.1", "val3");
+    properties.put("key3.1.1", "val4");
+
+    JobSpec jobSpec = JobSpec.builder("testJob")
+        .withConfig(ConfigUtils.propertiesToConfig(properties))
+        .withVersion("1")
+        .withDescription("")
+        .withTemplate(new URI("FS:///")).build();
     return jobSpec;
   }
 
@@ -68,6 +78,8 @@ public class FsSpecProducerTest {
     Assert.assertEquals(jobSpecs.get(0).getRight().getUri().toString(), "testJob");
     Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key1"), "val1");
     Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key2"), "val2");
+    Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key3.1" + ConfigUtils.STRIP_SUFFIX), "val3");
+    Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key3.1.1"), "val4");
   }
 
   @Test (dependsOnMethods = "testAddSpec")
@@ -80,6 +92,8 @@ public class FsSpecProducerTest {
     Assert.assertEquals(jobSpecs.get(0).getRight().getUri().toString(), "testJob");
     Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key1"), "val1");
     Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key2"), "val2");
+    Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key3.1" + ConfigUtils.STRIP_SUFFIX), "val3");
+    Assert.assertEquals(((JobSpec) jobSpecs.get(0).getRight()).getConfig().getString("key3.1.1"), "val4");
   }
 
   @Test (dependsOnMethods = "testUpdateSpec")