You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by le...@apache.org on 2020/03/11 01:22:32 UTC
[incubator-gobblin] branch master updated:
[Gobblin-1077][GOBBLIN-1077] Fix bug in HiveDataset.resolveConfig
This is an automated email from the ASF dual-hosted git repository.
lesun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 9444946 [Gobblin-1077][GOBBLIN-1077] Fix bug in HiveDataset.resolveConfig
9444946 is described below
commit 94449461353ac0fb11cf33c474c10870d7212280
Author: Zihan Li <zi...@zihli-mn1.linkedin.biz>
AuthorDate: Tue Mar 10 18:22:26 2020 -0700
[Gobblin-1077][GOBBLIN-1077] Fix bug in HiveDataset.resolveConfig
add writer schema to workUnitState
directly use writer.latest.schema
address comments
fix the bug in HiveDataset.resolveConfig
remove duplicated code in YarnService
reformat test code
address comments
Closes #2917 from ZihanLi58/GOBBLIN-1077
---
.../gobblin/data/management/copy/hive/HiveDataset.java | 9 +++++----
.../gobblin/data/management/copy/hive/HiveDatasetTest.java | 14 +++++++++-----
2 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java
index 26c7d7e..d550ec9 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java
@@ -17,6 +17,7 @@
package org.apache.gobblin.data.management.copy.hive;
+import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
@@ -266,7 +267,7 @@ public class HiveDataset implements PrioritizedCopyableDataset {
Preconditions.checkNotNull(realDbAndTable, "Real DB and table should not be null");
Preconditions.checkNotNull(logicalDbAndTable, "Logical DB and table should not be null");
- Properties resolvedProperties = new Properties();
+ ImmutableMap.Builder<String, Object> immutableMapBuilder = ImmutableMap.builder();
Config resolvedConfig = datasetConfig.resolve();
for (Map.Entry<String, ConfigValue> entry : resolvedConfig.entrySet()) {
if (ConfigValueType.LIST.equals(entry.getValue().valueType())) {
@@ -285,16 +286,16 @@ public class HiveDataset implements PrioritizedCopyableDataset {
}
listToStringWithQuotes.append("\"").append(resolvedValueStr).append("\"");
}
- resolvedProperties.setProperty(entry.getKey(), listToStringWithQuotes.toString());
+ immutableMapBuilder.put(entry.getKey(), listToStringWithQuotes.toString());
} else {
String resolvedValue = StringUtils.replaceEach(resolvedConfig.getString(entry.getKey()),
new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN },
new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() });
- resolvedProperties.setProperty(entry.getKey(), resolvedValue);
+ immutableMapBuilder.put(entry.getKey(), resolvedValue);
}
}
- return ConfigUtils.propertiesToConfig(resolvedProperties);
+ return ConfigFactory.parseMap(immutableMapBuilder.build());
}
/**
diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetTest.java
index ba0ad49..d9f3733 100644
--- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetTest.java
+++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/HiveDatasetTest.java
@@ -18,6 +18,7 @@ package org.apache.gobblin.data.management.copy.hive;
import java.io.IOException;
+import org.apache.gobblin.util.ConfigUtils;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -31,12 +32,14 @@ import org.apache.gobblin.data.management.retention.version.HiveDatasetVersionCl
public class HiveDatasetTest {
private static String DUMMY_CONFIG_KEY_WITH_DB_TOKEN = "dummyConfig.withDB";
+ private static String DUMMY_CONFIG_KEY_WITH_STRIP_SUFFIX = "dummyConfig" + ConfigUtils.STRIP_SUFFIX;
private static String DUMMY_CONFIG_KEY_WITH_TABLE_TOKEN = "dummyConfig.withTable";
- private static Config config = ConfigFactory.parseMap(ImmutableMap.<String, String> of(
- HiveDatasetVersionCleaner.REPLACEMENT_HIVE_DB_NAME_KEY, "resPrefix_$LOGICAL_DB_resPostfix",
- HiveDatasetVersionCleaner.REPLACEMENT_HIVE_TABLE_NAME_KEY, "resPrefix_$LOGICAL_TABLE_resPostfix",
- DUMMY_CONFIG_KEY_WITH_DB_TOKEN, "resPrefix_$DB_resPostfix",
- DUMMY_CONFIG_KEY_WITH_TABLE_TOKEN, "resPrefix_$TABLE_resPostfix"));
+ private static Config config = ConfigFactory.parseMap(ImmutableMap.<String,String> builder()
+ .put(DUMMY_CONFIG_KEY_WITH_STRIP_SUFFIX, "testRoot")
+ .put(HiveDatasetVersionCleaner.REPLACEMENT_HIVE_DB_NAME_KEY, "resPrefix_$LOGICAL_DB_resPostfix")
+ .put(HiveDatasetVersionCleaner.REPLACEMENT_HIVE_TABLE_NAME_KEY, "resPrefix_$LOGICAL_TABLE_resPostfix")
+ .put(DUMMY_CONFIG_KEY_WITH_DB_TOKEN, "resPrefix_$DB_resPostfix")
+ .put(DUMMY_CONFIG_KEY_WITH_TABLE_TOKEN, "resPrefix_$TABLE_resPostfix").build());
@Test
public void testParseLogicalDbAndTable() throws IOException {
@@ -162,5 +165,6 @@ public class HiveDatasetTest {
"resPrefix_logicalDb_resPostfix", "Logical DB not resolved correctly");
Assert.assertEquals(resolvedConfig.getString(HiveDatasetVersionCleaner.REPLACEMENT_HIVE_TABLE_NAME_KEY),
"resPrefix_logicalTable_resPostfix", "Logical Table not resolved correctly");
+ Assert.assertEquals(resolvedConfig.getString(DUMMY_CONFIG_KEY_WITH_STRIP_SUFFIX),"testRoot");
}
}