You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2019/08/16 04:50:49 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-857] Extending
getTopicsFromConfigStore to accept topicName directly
This is an automated email from the ASF dual-hosted git repository.
hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 94dd2ae [GOBBLIN-857] Extending getTopicsFromConfigStore to accept topicName directly
94dd2ae is described below
commit 94dd2ae9566bd76fb11c68850a8fcc6e64797824
Author: autumnust <le...@linkedin.com>
AuthorDate: Thu Aug 15 21:50:42 2019 -0700
[GOBBLIN-857] Extending getTopicsFromConfigStore to accept topicName directly
Closes #2713 from autumnust/extendConfigStoreUtils
---
.../store/api/PhysicalPathNotExistException.java | 43 +++++++++++++++++++++
.../config/store/zip/ZipFileConfigStore.java | 30 +++++++-------
.../config/store/zip/ZipFileConfigStoreTest.java | 1 -
.../src/test/resources/zipStoreTest.zip | Bin 2990 -> 2642 bytes
.../extractor/extract/kafka/ConfigStoreUtils.java | 19 +++++++--
.../extract/kafka/ZipConfigStoreUtilsTest.java | 23 +++++++++--
.../src/test/resources/IvyConfigStoreTest.zip | Bin 8389 -> 9450 bytes
.../v1.0/data/tracking/Topic3/.DS_Store | Bin 0 -> 6148 bytes
.../v1.0/data/tracking/Topic3/includes.conf | 1 +
.../v1.0/data/tracking/Topic3/main.conf | 1 +
.../_CONFIG_STORE/v1.0/tags/random/main.conf | 0
11 files changed, 93 insertions(+), 25 deletions(-)
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/api/PhysicalPathNotExistException.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/api/PhysicalPathNotExistException.java
new file mode 100644
index 0000000..355fe80
--- /dev/null
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/api/PhysicalPathNotExistException.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.config.store.api;
+
+import java.net.URI;
+
+
+/**
+ * Indicating the physical layer doesn't exist in certain scenarios (e.g. When root is not detected which usually
+ * is due to users making mistakes on configuration.
+ */
+public class PhysicalPathNotExistException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+ private static final String MESSAGE_FORMAT =
+ "Failed to find the location %s in config store because of: %s ";
+
+ private final URI uri;
+
+ public PhysicalPathNotExistException(URI inspectedURI, String errorMessage) {
+ super(String.format(MESSAGE_FORMAT, inspectedURI, errorMessage));
+ this.uri = inspectedURI;
+ }
+
+ public URI getUri() {
+ return this.uri;
+ }
+}
\ No newline at end of file
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java
index 711a836..72c046a 100644
--- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java
@@ -27,10 +27,15 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import java.util.concurrent.ExecutionException;
import java.util.stream.Stream;
import org.apache.commons.lang.StringUtils;
+import org.apache.gobblin.config.common.impl.SingleLinkedListConfigKeyPath;
+import org.apache.gobblin.config.store.api.ConfigKeyPath;
+import org.apache.gobblin.config.store.api.ConfigStore;
+import org.apache.gobblin.config.store.api.PhysicalPathNotExistException;
+import org.apache.gobblin.config.store.api.VersionDoesNotExistException;
+import org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore;
import com.google.common.base.Charsets;
import com.google.common.base.Optional;
@@ -39,12 +44,6 @@ import com.sun.nio.zipfs.ZipFileSystem;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
-import org.apache.gobblin.config.common.impl.SingleLinkedListConfigKeyPath;
-import org.apache.gobblin.config.store.api.ConfigKeyPath;
-import org.apache.gobblin.config.store.api.ConfigStore;
-import org.apache.gobblin.config.store.api.VersionDoesNotExistException;
-import org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore;
-
import lombok.extern.slf4j.Slf4j;
@@ -103,12 +102,12 @@ public class ZipFileConfigStore implements ConfigStore {
Preconditions.checkArgument(version.equals(getCurrentVersion()));
List<ConfigKeyPath> children = new ArrayList<>();
- Path datasetDir = getDatasetDirForKey(configKey, version);
+ Path datasetDir = getDatasetDirForKey(configKey);
try {
-
if (!Files.exists(this.fs.getPath(datasetDir.toString()))) {
- return children;
+ throw new PhysicalPathNotExistException(this.logicalStoreRoot,
+ "Cannot find physical location:" + this.fs.getPath(datasetDir.toString()));
}
Stream<Path> files = Files.walk(datasetDir, 1);
@@ -143,7 +142,7 @@ public class ZipFileConfigStore implements ConfigStore {
Preconditions.checkArgument(version.equals(getCurrentVersion()));
List<ConfigKeyPath> configKeyPaths = new ArrayList<>();
- Path datasetDir = getDatasetDirForKey(configKey, version);
+ Path datasetDir = getDatasetDirForKey(configKey);
Path includesFile = this.fs.getPath(datasetDir.toString(), SimpleHadoopFilesystemConfigStore.INCLUDES_CONF_FILE_NAME);
try {
@@ -172,7 +171,7 @@ public class ZipFileConfigStore implements ConfigStore {
Preconditions.checkNotNull(configKey, "configKey cannot be null!");
Preconditions.checkArgument(version.equals(getCurrentVersion()));
- Path datasetDir = getDatasetDirForKey(configKey, version);
+ Path datasetDir = getDatasetDirForKey(configKey);
Path mainConfFile = this.fs.getPath(datasetDir.toString(), SimpleHadoopFilesystemConfigStore.MAIN_CONF_FILE_NAME);
try {
@@ -191,10 +190,7 @@ public class ZipFileConfigStore implements ConfigStore {
}
}
- /**
- * Get path object using zipped file system and relative path
- */
- private Path getDatasetDirForKey(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException {
- return this.fs.getPath(this.storePrefix, version, configKey.getAbsolutePathString());
+ private Path getDatasetDirForKey(ConfigKeyPath configKey) throws VersionDoesNotExistException {
+ return this.fs.getPath(this.storePrefix, configKey.getAbsolutePathString());
}
}
diff --git a/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java b/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java
index 12f69ca..87bc491 100644
--- a/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java
+++ b/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java
@@ -52,7 +52,6 @@ public class ZipFileConfigStoreTest {
/**
* Layout of testing config store:
* /_CONFIG_STORE
- * /testVersion
* /test
* /child1
* main.conf (gobblin.test.property = "string2")
diff --git a/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip b/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip
index 6f8896d..fa31885 100644
Binary files a/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip and b/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip differ
diff --git a/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/ConfigStoreUtils.java b/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/ConfigStoreUtils.java
index 6346d64..a4b0c4f 100644
--- a/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/ConfigStoreUtils.java
+++ b/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/ConfigStoreUtils.java
@@ -94,8 +94,21 @@ public class ConfigStoreUtils {
return topicUri;
}
+ /**
+ * Used when topic name needs to be fetched from Properties object, assuming we knew the topicKey.
+ */
public static Optional<Config> getConfigForTopic(Properties properties, String topicKey, ConfigClient configClient) {
- Optional<String> configStoreUri = getConfigStoreUri(properties);
+ Preconditions.checkArgument(properties.containsKey(topicKey), "Missing required property " + topicKey);
+ String topicName = properties.getProperty(topicKey);
+
+ return getConfigForTopicWithName(properties, topicName, configClient);
+ }
+
+ /**
+ * Used when topic name is known.
+ */
+ public static Optional<Config> getConfigForTopicWithName(Properties properties, String topicName, ConfigClient configClient) {
+ Optional<String> configStoreUri = ConfigStoreUtils.getConfigStoreUri(properties);
Optional<Config> config = Optional.<Config>absent();
if (!configStoreUri.isPresent()) {
return config;
@@ -103,11 +116,9 @@ public class ConfigStoreUtils {
try {
Preconditions.checkArgument(properties.containsKey(GOBBLIN_CONFIG_COMMONPATH),
"Missing required property " + GOBBLIN_CONFIG_COMMONPATH);
- Preconditions.checkArgument(properties.containsKey(topicKey), "Missing required property " + topicKey);
- String topicName = properties.getProperty(topicKey);
String commonPath = properties.getProperty(GOBBLIN_CONFIG_COMMONPATH);
config = Optional.fromNullable(
- getConfig(configClient, getUriStringForTopic(topicName, commonPath, configStoreUri.get()),
+ ConfigStoreUtils.getConfig(configClient, ConfigStoreUtils.getUriStringForTopic(topicName, commonPath, configStoreUri.get()),
ConfigClientUtils.getOptionalRuntimeConfig(properties)));
} catch (URISyntaxException e) {
log.error("Unable to get config", e);
diff --git a/gobblin-modules/gobblin-kafka-common/src/test/java/org/apache/gobblin/source/extractor/extract/kafka/ZipConfigStoreUtilsTest.java b/gobblin-modules/gobblin-kafka-common/src/test/java/org/apache/gobblin/source/extractor/extract/kafka/ZipConfigStoreUtilsTest.java
index ebdd27b..3a9e350 100644
--- a/gobblin-modules/gobblin-kafka-common/src/test/java/org/apache/gobblin/source/extractor/extract/kafka/ZipConfigStoreUtilsTest.java
+++ b/gobblin-modules/gobblin-kafka-common/src/test/java/org/apache/gobblin/source/extractor/extract/kafka/ZipConfigStoreUtilsTest.java
@@ -54,6 +54,10 @@ import static org.mockito.Matchers.anyList;
/**
* The same testing routine for ivy-based config-store (ZipConfigStore)
* Make sure everything inside {@link ConfigStoreUtils} will work for {@link ZipFileConfigStore} implementation.
+ *
+ * Note that {@link ZipFileConfigStore}, doesn't contain version folder. More specifically, under .zip file
+ * there would be configNodes directly, unlike {@link org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore}
+ * where there would be a version folder inside the configStore root path.
*/
public class ZipConfigStoreUtilsTest {
private String configStoreUri;
@@ -75,19 +79,32 @@ public class ZipConfigStoreUtilsTest {
}
@Test
- public void testGetListOfTopicNamesByFilteringTag()
- throws Exception {
+ public void testGetListOfTopicNamesByFilteringTag() {
Properties properties = new Properties();
properties.setProperty(GOBBLIN_CONFIG_TAGS_WHITELIST, "/tags/whitelist");
properties.setProperty(GOBBLIN_CONFIG_FILTER, "/data/tracking");
properties.setProperty(GOBBLIN_CONFIG_COMMONPATH, "/data/tracking");
-
List<String> result = ConfigStoreUtils
.getListOfTopicNamesByFilteringTag(properties, configClient, Optional.absent(), configStoreUri,
GOBBLIN_CONFIG_TAGS_WHITELIST);
Assert.assertEquals(result.size(), 2);
Assert.assertTrue(result.contains("Topic1"));
Assert.assertTrue(result.contains("Topic2"));
+
+ properties.setProperty(GOBBLIN_CONFIG_TAGS_WHITELIST, "/tags/random");
+ result = ConfigStoreUtils
+ .getListOfTopicNamesByFilteringTag(properties, configClient, Optional.absent(), configStoreUri,
+ GOBBLIN_CONFIG_TAGS_WHITELIST);
+ Assert.assertEquals(result.size(), 1);
+ Assert.assertTrue(result.contains("Topic3"));
+
+ properties.setProperty(GOBBLIN_CONFIG_TAGS_BLACKLIST, "/tags/blacklist");
+ result = ConfigStoreUtils
+ .getListOfTopicNamesByFilteringTag(properties, configClient, Optional.absent(), configStoreUri,
+ GOBBLIN_CONFIG_TAGS_BLACKLIST);
+ Assert.assertEquals(result.size(), 2);
+ Assert.assertTrue(result.contains("Topic1"));
+ Assert.assertTrue(result.contains("Topic2"));
}
@Test
diff --git a/gobblin-modules/gobblin-kafka-common/src/test/resources/IvyConfigStoreTest.zip b/gobblin-modules/gobblin-kafka-common/src/test/resources/IvyConfigStoreTest.zip
index 41fcac2..b46c5a4 100644
Binary files a/gobblin-modules/gobblin-kafka-common/src/test/resources/IvyConfigStoreTest.zip and b/gobblin-modules/gobblin-kafka-common/src/test/resources/IvyConfigStoreTest.zip differ
diff --git a/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/.DS_Store b/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/.DS_Store
new file mode 100644
index 0000000..5008ddf
Binary files /dev/null and b/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/.DS_Store differ
diff --git a/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/includes.conf b/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/includes.conf
new file mode 100644
index 0000000..6a948fa
--- /dev/null
+++ b/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/includes.conf
@@ -0,0 +1 @@
+/tags/random
\ No newline at end of file
diff --git a/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/main.conf b/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/main.conf
new file mode 100644
index 0000000..985e7e2
--- /dev/null
+++ b/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/data/tracking/Topic3/main.conf
@@ -0,0 +1 @@
+aaaaa=bbbb
\ No newline at end of file
diff --git a/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/tags/random/main.conf b/gobblin-modules/gobblin-kafka-common/src/test/resources/_CONFIG_STORE/v1.0/tags/random/main.conf
new file mode 100644
index 0000000..e69de29