You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2018/09/11 16:10:53 UTC
incubator-gobblin git commit: [GOBBLIN-567] Create config store that
downloads and reads from a local jar
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 0ae37a849 -> 5eee52d93
[GOBBLIN-567] Create config store that downloads and reads from a local jar
Closes #2430 from jack-moseley/zip-config-store
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/5eee52d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/5eee52d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/5eee52d9
Branch: refs/heads/master
Commit: 5eee52d93ffe663c1b6c7de3a0e3f2d9898708e1
Parents: 0ae37a8
Author: Jack Moseley <jm...@linkedin.com>
Authored: Tue Sep 11 09:10:47 2018 -0700
Committer: Hung Tran <hu...@linkedin.com>
Committed: Tue Sep 11 09:10:47 2018 -0700
----------------------------------------------------------------------
.../hdfs/SimpleHDFSConfigStoreFactory.java | 8 +-
.../store/hdfs/SimpleHDFSStoreMetadata.java | 4 +-
.../hdfs/SimpleHadoopFilesystemConfigStore.java | 30 ++-
.../config/store/zip/IvyConfigStoreFactory.java | 116 +++++++++++
.../config/store/zip/ZipFileConfigStore.java | 199 +++++++++++++++++++
....gobblin.config.store.api.ConfigStoreFactory | 1 +
.../store/zip/ZipFileConfigStoreTest.java | 101 ++++++++++
.../src/test/resources/zipStoreTest.zip | Bin 0 -> 1821 bytes
gobblin-utility/build.gradle | 1 +
.../org/apache/gobblin/util/DownloadUtils.java | 96 +++++++++
gradle/scripts/dependencyDefinitions.gradle | 1 +
11 files changed, 537 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java
index b20d1e2..cfa36cf 100644
--- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java
@@ -18,20 +18,14 @@ package org.apache.gobblin.config.store.hdfs;
import java.io.IOException;
import java.net.URI;
-import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
-import com.google.common.base.Strings;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
-import org.apache.gobblin.config.store.api.ConfigStoreCreationException;
import org.apache.gobblin.config.store.api.ConfigStoreFactory;
import org.apache.gobblin.util.ConfigUtils;
@@ -44,7 +38,7 @@ import org.apache.gobblin.util.ConfigUtils;
*/
public class SimpleHDFSConfigStoreFactory extends SimpleHadoopFilesystemConfigStoreFactory {
- protected static final String HDFS_SCHEME_NAME = "hdfs";
+ public static final String HDFS_SCHEME_NAME = "hdfs";
/** Instantiates a new instance using standard typesafe config defaults:
* {@link ConfigFactory#load()} */
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java
index 98d79d2..d196449 100644
--- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java
@@ -55,7 +55,7 @@ public class SimpleHDFSStoreMetadata {
* @param fs where metadata is stored
* @param configStoreDir path to {@link SimpleHadoopFilesystemConfigStore#CONFIG_STORE_NAME}
*/
- SimpleHDFSStoreMetadata(final FileSystem fs, final Path configStoreDir) {
+ public SimpleHDFSStoreMetadata(final FileSystem fs, final Path configStoreDir) {
this.storeMetadataFilePath = new Path(configStoreDir, CONFIG_STORE_METADATA_FILENAME);
this.fs = fs;
}
@@ -119,7 +119,7 @@ public class SimpleHDFSStoreMetadata {
* Get the current version from {@link #CONFIG_STORE_METADATA_FILENAME} file at {@link #storeMetadataFilePath}
*
*/
- String getCurrentVersion() throws IOException {
+ public String getCurrentVersion() throws IOException {
return readMetadata().getString(CONFIG_STORE_METADATA_CURRENT_VERSION_KEY);
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java
index da541e2..4008065 100644
--- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java
@@ -122,10 +122,10 @@ import org.apache.gobblin.util.io.StreamUtils;
@ConfigStoreWithStableVersioning
public class SimpleHadoopFilesystemConfigStore implements ConfigStore, Deployable<FsDeploymentConfig> {
- protected static final String CONFIG_STORE_NAME = "_CONFIG_STORE";
+ public static final String CONFIG_STORE_NAME = "_CONFIG_STORE";
- private static final String MAIN_CONF_FILE_NAME = "main.conf";
- private static final String INCLUDES_CONF_FILE_NAME = "includes.conf";
+ public static final String MAIN_CONF_FILE_NAME = "main.conf";
+ public static final String INCLUDES_CONF_FILE_NAME = "includes.conf";
private static final String INCLUDES_KEY_NAME = "includes";
private final FileSystem fs;
@@ -265,14 +265,7 @@ public class SimpleHadoopFilesystemConfigStore implements ConfigStore, Deployabl
FileStatus includesFileStatus = this.fs.getFileStatus(includesFile);
if (!includesFileStatus.isDirectory()) {
try (InputStream includesConfInStream = this.fs.open(includesFileStatus.getPath())) {
- /*
- * The includes returned are used to build a fallback chain.
- * With the natural order, if a key found in the first include it is not be overriden by the next include.
- * By reversing the list, the Typesafe fallbacks are constructed bottom up.
- */
- configKeyPaths.addAll(Lists.newArrayList(
- Iterables.transform(Lists.reverse(resolveIncludesList(IOUtils.readLines(includesConfInStream, Charsets.UTF_8), runtimeConfig)),
- new IncludesToConfigKey())));
+ configKeyPaths.addAll(getResolvedConfigKeyPaths(includesConfInStream, runtimeConfig));
}
}
} catch (IOException e) {
@@ -283,6 +276,21 @@ public class SimpleHadoopFilesystemConfigStore implements ConfigStore, Deployabl
}
/**
+ * Get resolved config key paths given an includes file as an {@link InputStream}
+ *
+ * The includes returned are used to build a fallback chain.
+ * With the natural order, if a key found in the first include it is not be overriden by the next include.
+ * By reversing the list, the Typesafe fallbacks are constructed bottom up.
+ *
+ * @param includesConfInStream includes.conf file as an {@link InputStream}
+ * @return a {@link List} of resolved ConfigKeyPaths
+ */
+ public static List<ConfigKeyPath> getResolvedConfigKeyPaths(InputStream includesConfInStream, Optional<Config> runtimeConfig) throws IOException {
+ return Lists.newArrayList(Iterables.transform(Lists.reverse(resolveIncludesList(
+ IOUtils.readLines(includesConfInStream, Charsets.UTF_8), runtimeConfig)), new IncludesToConfigKey()));
+ }
+
+ /**
* A helper to resolve System properties and Environment variables in includes paths
* The method loads the list of unresolved <code>includes</code> into an in-memory {@link Config} object and reolves
* with a fallback on {@link ConfigFactory#defaultOverrides()}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java
new file mode 100644
index 0000000..dba7d3a
--- /dev/null
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.config.store.zip;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Paths;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+
+import com.sun.nio.zipfs.ZipFileSystem;
+
+import org.apache.gobblin.config.store.api.ConfigStoreCreationException;
+import org.apache.gobblin.config.store.api.ConfigStoreFactory;
+import org.apache.gobblin.config.store.hdfs.SimpleHDFSConfigStoreFactory;
+import org.apache.gobblin.config.store.hdfs.SimpleHDFSStoreMetadata;
+import org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore;
+import org.apache.gobblin.util.DownloadUtils;
+
+
+/**
+ * {@link ConfigStoreFactory} that downloads a jar file containing the config store paths through ivy and creates a
+ * {@link ZipFileConfigStore} with it. May be useful to avoid making many HDFS calls for large config stores.
+ *
+ * An ivy settings file must be present on the classpath named {@link DownloadUtils#IVY_SETTINGS_FILE_NAME}
+ */
+public class IvyConfigStoreFactory implements ConfigStoreFactory<ZipFileConfigStore> {
+
+ private static final String IVY_SCHEME_PREFIX = "ivy-";
+ private static final String ORG_KEY = "org";
+ private static final String MODULE_KEY = "module";
+ private static final String STORE_PREFIX_KEY = "storePrefix";
+
+ @Override
+ public String getScheme() {
+ return getSchemePrefix() + SimpleHDFSConfigStoreFactory.HDFS_SCHEME_NAME;
+ }
+
+ public String getSchemePrefix() {
+ return IVY_SCHEME_PREFIX;
+ }
+
+ /**
+ * Example configKey URI (configuration is passed as part of the query)
+ *
+ * ivy-hdfs://<hdfsURI>/path/to/config/store?org=<jarOrg>&module=<jarModule>&storePrefix=_CONFIG_STORE
+ *
+ * ivy-hdfs: scheme for this factory
+ * hdfsURI/path/to/config/store: location of HDFS config store (used for getting current version)
+ * org/module: org and module of jar containing config store
+ * storePrefix: prefix to paths in config store
+ */
+ @Override
+ public ZipFileConfigStore createConfigStore(URI configKey) throws ConfigStoreCreationException {
+ if (!configKey.getScheme().equals(getScheme())) {
+ throw new ConfigStoreCreationException(configKey, "Config key URI must have scheme " + getScheme());
+ }
+
+ Properties factoryProps = new Properties();
+ for (NameValuePair param : URLEncodedUtils.parse(configKey, "UTF-8")) {
+ factoryProps.setProperty(param.getName(), param.getValue());
+ }
+
+ String jarOrg = factoryProps.getProperty(ORG_KEY);
+ String jarModule = factoryProps.getProperty(MODULE_KEY);
+
+ if (jarOrg == null || jarModule == null) {
+ throw new ConfigStoreCreationException(configKey, "Config key URI must contain org and module to download from");
+ }
+
+ try {
+ SimpleHDFSStoreMetadata metadata = new SimpleHDFSStoreMetadata(
+ org.apache.hadoop.fs.FileSystem.get(new Configuration()), new Path(configKey.getPath(),
+ SimpleHadoopFilesystemConfigStore.CONFIG_STORE_NAME));
+ String currentVersion = metadata.getCurrentVersion();
+
+ URI[] uris = DownloadUtils.downloadJar(jarOrg, jarModule, currentVersion, false);
+
+ if (uris.length != 1) {
+ throw new ConfigStoreCreationException(configKey, "Expected one jar file from URI");
+ }
+
+ FileSystem zipFs = FileSystems.newFileSystem(Paths.get(uris[0].getPath()), null);
+
+ if (!(zipFs instanceof ZipFileSystem)) {
+ throw new ConfigStoreCreationException(configKey, "Downloaded file must be a zip or jar file");
+ }
+
+ return new ZipFileConfigStore((ZipFileSystem) zipFs, configKey, currentVersion, factoryProps.getProperty(STORE_PREFIX_KEY, ""));
+ } catch (IOException e) {
+ throw new ConfigStoreCreationException(configKey, e);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java
new file mode 100644
index 0000000..5475b49
--- /dev/null
+++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gobblin.config.store.zip;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.sun.nio.zipfs.ZipFileSystem;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+
+import org.apache.gobblin.config.common.impl.SingleLinkedListConfigKeyPath;
+import org.apache.gobblin.config.store.api.ConfigKeyPath;
+import org.apache.gobblin.config.store.api.ConfigStore;
+import org.apache.gobblin.config.store.api.VersionDoesNotExistException;
+import org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore;
+
+import lombok.extern.slf4j.Slf4j;
+
+
+/**
+ * {@link ConfigStore} that uses a zipped file containing all the config store paths.
+ *
+ * Similar to {@link SimpleHadoopFilesystemConfigStore} but using java APIs instead of Hadoop APIs for the Filesystem to
+ * allow reading the file without unzipping.
+ *
+ * It is assumed that the version passed in the constructor will be the only version used.
+ */
+@Slf4j
+public class ZipFileConfigStore implements ConfigStore {
+
+ private final FileSystem fs;
+ private final URI logicalStoreRoot;
+ private String version;
+ private String storePrefix;
+
+ /**
+ * Construct a ZipFileConfigStore
+ *
+ * @param fs A {@link ZipFileSystem} created using the zip file or jar containing the config store
+ * @param logicalStoreRoot URI of this config store's root
+ * @param version Config store version to use (only version allowed for lookups is the version passed here)
+ * @param storePrefix Prefix to use if all paths in config store are under a parent directory
+ */
+ public ZipFileConfigStore(ZipFileSystem fs, URI logicalStoreRoot, String version, String storePrefix) {
+ Preconditions.checkNotNull(fs);
+ Preconditions.checkNotNull(logicalStoreRoot);
+ Preconditions.checkNotNull(version);
+
+ this.fs = fs;
+ this.logicalStoreRoot = logicalStoreRoot;
+ this.version = version;
+ this.storePrefix = storePrefix;
+ }
+
+ @Override
+ public String getCurrentVersion() {
+ return this.version;
+ }
+
+ @Override
+ public URI getStoreURI() {
+ return this.logicalStoreRoot;
+ }
+
+ /**
+ * Retrieves all the children of the given {@link ConfigKeyPath} using {@link Files#walk} to list files
+ */
+ @Override
+ public Collection<ConfigKeyPath> getChildren(ConfigKeyPath configKey, String version)
+ throws VersionDoesNotExistException {
+ Preconditions.checkNotNull(configKey, "configKey cannot be null!");
+ Preconditions.checkArgument(version.equals(getCurrentVersion()));
+
+ List<ConfigKeyPath> children = new ArrayList<>();
+ Path datasetDir = getDatasetDirForKey(configKey);
+
+ try {
+
+ if (!Files.exists(this.fs.getPath(datasetDir.toString()))) {
+ return children;
+ }
+
+ Stream<Path> files = Files.walk(datasetDir, 1);
+
+ for (Iterator<Path> it = files.iterator(); it.hasNext();) {
+ Path path = it.next();
+
+ if (Files.isDirectory(path) && !path.equals(datasetDir)) {
+ children.add(configKey.createChild(StringUtils.removeEnd(path.getName(path.getNameCount() - 1).toString(),
+ SingleLinkedListConfigKeyPath.PATH_DELIMETER)));
+ }
+
+ }
+ return children;
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("Error while getting children for configKey: \"%s\"", configKey), e);
+ }
+ }
+
+ /**
+ * Retrieves all the {@link ConfigKeyPath}s that are imported by the given {@link ConfigKeyPath}. Similar to
+ * {@link SimpleHadoopFilesystemConfigStore#getOwnImports}
+ */
+ @Override
+ public List<ConfigKeyPath> getOwnImports(ConfigKeyPath configKey, String version) {
+ return getOwnImports(configKey, version, Optional.<Config>absent());
+ }
+
+ public List<ConfigKeyPath> getOwnImports(ConfigKeyPath configKey, String version, Optional<Config> runtimeConfig)
+ throws VersionDoesNotExistException {
+ Preconditions.checkNotNull(configKey, "configKey cannot be null!");
+ Preconditions.checkArgument(version.equals(getCurrentVersion()));
+
+ List<ConfigKeyPath> configKeyPaths = new ArrayList<>();
+ Path datasetDir = getDatasetDirForKey(configKey);
+ Path includesFile = this.fs.getPath(datasetDir.toString(), SimpleHadoopFilesystemConfigStore.INCLUDES_CONF_FILE_NAME);
+
+ try {
+ if (!Files.exists(includesFile)) {
+ return configKeyPaths;
+ }
+
+ if (!Files.isDirectory(includesFile)) {
+ try (InputStream includesConfInStream = Files.newInputStream(includesFile)) {
+ configKeyPaths = SimpleHadoopFilesystemConfigStore.getResolvedConfigKeyPaths(includesConfInStream, runtimeConfig);
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("Error while getting config for configKey: \"%s\"", configKey), e);
+ }
+
+ return configKeyPaths;
+ }
+
+ /**
+ * Retrieves the {@link Config} for the given {@link ConfigKeyPath}. Similar to
+ * {@link SimpleHadoopFilesystemConfigStore#getOwnConfig}
+ */
+ @Override
+ public Config getOwnConfig(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException {
+ Preconditions.checkNotNull(configKey, "configKey cannot be null!");
+ Preconditions.checkArgument(version.equals(getCurrentVersion()));
+
+ Path datasetDir = getDatasetDirForKey(configKey);
+ Path mainConfFile = this.fs.getPath(datasetDir.toString(), SimpleHadoopFilesystemConfigStore.MAIN_CONF_FILE_NAME);
+
+ try {
+ if (!Files.exists(mainConfFile)) {
+ return ConfigFactory.empty();
+ }
+
+ if (!Files.isDirectory(mainConfFile)) {
+ try (InputStream mainConfInputStream = Files.newInputStream(mainConfFile)) {
+ return ConfigFactory.parseReader(new InputStreamReader(mainConfInputStream, Charsets.UTF_8));
+ }
+ }
+ return ConfigFactory.empty();
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("Error while getting config for configKey: \"%s\"", configKey), e);
+ }
+ }
+
+ /**
+ * Get path object using zipped file system and relative path
+ */
+ private Path getDatasetDirForKey(ConfigKeyPath configKey) throws VersionDoesNotExistException {
+ return this.fs.getPath(this.storePrefix, configKey.getAbsolutePathString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory b/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory
index cf3359a..69865c1 100644
--- a/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory
+++ b/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory
@@ -18,3 +18,4 @@
org.apache.gobblin.config.store.hdfs.SimpleHDFSConfigStoreFactory
org.apache.gobblin.config.store.hdfs.SimpleLocalHDFSConfigStoreFactory
org.apache.gobblin.config.store.hdfs.DefaultCapableLocalConfigStoreFactory
+org.apache.gobblin.config.store.zip.IvyConfigStoreFactory
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java b/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java
new file mode 100644
index 0000000..70ff132
--- /dev/null
+++ b/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gobblin.config.store.zip;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import com.sun.nio.zipfs.ZipFileSystem;
+import com.typesafe.config.Config;
+
+import org.apache.gobblin.config.common.impl.SingleLinkedListConfigKeyPath;
+import org.apache.gobblin.config.store.api.ConfigKeyPath;
+import org.apache.gobblin.config.store.api.ConfigStoreCreationException;
+
+
+/**
+ * Unit tests for {@link ZipFileConfigStore}
+ */
+@Test
+public class ZipFileConfigStoreTest {
+
+ private ZipFileConfigStore store;
+ private String version = "testVersion";
+ private ConfigKeyPath rootPath = SingleLinkedListConfigKeyPath.ROOT;
+ private ConfigKeyPath testPath = rootPath.createChild("test");
+ private ConfigKeyPath child1Path = testPath.createChild("child1");
+ private ConfigKeyPath child2Path = testPath.createChild("child2");
+
+ @BeforeClass
+ public void setUp() throws URISyntaxException, ConfigStoreCreationException, IOException {
+ Path path = Paths.get(this.getClass().getClassLoader().getResource("zipStoreTest.zip").getPath());
+ FileSystem fs = FileSystems.newFileSystem(path, null);
+
+ this.store = new ZipFileConfigStore((ZipFileSystem) fs, path.toUri(), this.version, "_CONFIG_STORE");
+ }
+
+ @Test
+ public void testGetOwnConfig() {
+ Config config1 = this.store.getOwnConfig(this.rootPath, this.version);
+ Assert.assertEquals(config1.getString("gobblin.property.test1"), "prop1");
+ Assert.assertEquals(config1.getString("gobblin.property.test2"), "prop2");
+
+ Config config2 = this.store.getOwnConfig(this.testPath, this.version);
+ Assert.assertEquals(config2.getString("gobblin.test.property"), "string1");
+
+ Config config3 = this.store.getOwnConfig(this.child1Path, this.version);
+ Assert.assertEquals(config3.getString("gobblin.test.property"), "string2");
+
+ Config config4 = this.store.getOwnConfig(this.child2Path, this.version);
+ Assert.assertEquals(config4.getString("gobblin.test.property"), "string3");
+
+ }
+
+ @Test
+ public void testGetOwnImports() {
+ Collection<ConfigKeyPath> imports1 = this.store.getOwnImports(this.child1Path, this.version);
+ Assert.assertEquals(imports1.size(), 1);
+ Assert.assertTrue(imports1.contains(this.child1Path));
+
+ Collection<ConfigKeyPath> imports2 = this.store.getOwnImports(this.child2Path, this.version);
+ Assert.assertEquals(imports2.size(), 0);
+ }
+
+ @Test
+ public void testGetChildren() {
+ Collection<ConfigKeyPath> children1 = this.store.getChildren(this.rootPath, this.version);
+ Assert.assertEquals(children1.size(), 1);
+ Assert.assertTrue(children1.contains(this.testPath));
+
+ Collection<ConfigKeyPath> children2 = this.store.getChildren(this.testPath, this.version);
+ Assert.assertEquals(children2.size(), 2);
+ Assert.assertTrue(children2.contains(this.child1Path));
+ Assert.assertTrue(children2.contains(this.child2Path));
+
+ Collection<ConfigKeyPath> children3 = this.store.getChildren(this.child1Path, this.version);
+ Assert.assertEquals(children3.size(), 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip
----------------------------------------------------------------------
diff --git a/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip b/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip
new file mode 100644
index 0000000..ffe9603
Binary files /dev/null and b/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip differ
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-utility/build.gradle
----------------------------------------------------------------------
diff --git a/gobblin-utility/build.gradle b/gobblin-utility/build.gradle
index a8f87bb..ad49820 100644
--- a/gobblin-utility/build.gradle
+++ b/gobblin-utility/build.gradle
@@ -46,6 +46,7 @@ dependencies {
compile externalDependency.gson
compile externalDependency.opencsv
compile externalDependency.hadoopHdfs
+ compile externalDependency.groovy
runtime externalDependency.hadoopCommon
runtime externalDependency.hadoopClientCore
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java
----------------------------------------------------------------------
diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java
new file mode 100644
index 0000000..ab5468f
--- /dev/null
+++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.util;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URL;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import com.google.common.io.Resources;
+
+import groovy.grape.Grape;
+import groovy.lang.GroovyClassLoader;
+
+
+/**
+ * Utility class for downloads using grape
+ */
+public class DownloadUtils {
+ public static final String IVY_SETTINGS_FILE_NAME = "ivysettings.xml";
+
+ /**
+ * Download jar through {@link Grape} given an org, module and version
+ * It is assumed that an ivy settings file exists on the classpath
+ */
+ public static URI[] downloadJar(String org, String module, String version, boolean transitive) throws IOException {
+ Map<String, Object> artifactMap = Maps.newHashMap();
+ artifactMap.put("org", org);
+ artifactMap.put("module", module);
+ artifactMap.put("version", version);
+ artifactMap.put("transitive", transitive);
+ return downloadJar(artifactMap);
+ }
+
+ public static URI[] downloadJar(Map<String, Object> artifactMap) throws IOException {
+ System.setProperty("grape.config", getIvySettingsFile().getAbsolutePath());
+
+ Map<String, Object> args = Maps.newHashMap();
+ args.put("classLoader", AccessController.doPrivileged(new PrivilegedAction<GroovyClassLoader>() {
+ @Override
+ public GroovyClassLoader run() {
+ return new GroovyClassLoader();
+ }
+ }));
+ return Grape.resolve(args, artifactMap);
+ }
+
+ /**
+ * Get ivy settings file from classpath
+ */
+ public static File getIvySettingsFile() throws IOException {
+ URL settingsUrl = Thread.currentThread().getContextClassLoader().getResource(IVY_SETTINGS_FILE_NAME);
+ if (settingsUrl == null) {
+ throw new IOException("Failed to find " + IVY_SETTINGS_FILE_NAME + " from class path");
+ }
+
+ // Check if settingsUrl is file on classpath
+ File ivySettingsFile = new File(settingsUrl.getFile());
+ if (ivySettingsFile.exists()) {
+ // can access settingsUrl as a file
+ return ivySettingsFile;
+ }
+
+ // Create temporary Ivy settings file.
+ ivySettingsFile = File.createTempFile("ivy.settings", ".xml");
+ ivySettingsFile.deleteOnExit();
+
+ try (OutputStream os = new BufferedOutputStream(new FileOutputStream(ivySettingsFile))) {
+ Resources.copy(settingsUrl, os);
+ }
+
+ return ivySettingsFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gradle/scripts/dependencyDefinitions.gradle
----------------------------------------------------------------------
diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle
index c1f53db..f82b7e2 100644
--- a/gradle/scripts/dependencyDefinitions.gradle
+++ b/gradle/scripts/dependencyDefinitions.gradle
@@ -48,6 +48,7 @@ ext.externalDependency = [
"datanucleusRdbms": "org.datanucleus:datanucleus-rdbms:3.2.9",
"eventhub": "com.microsoft.azure:azure-eventhubs:0.9.0",
"guava": "com.google.guava:guava:15.0",
+ "groovy": "org.codehaus.groovy:groovy:2.4.8",
"gson": "com.google.code.gson:gson:2.6.2",
"findBugsAnnotations": "com.google.code.findbugs:jsr305:" + findBugsVersion,
"hadoopCommon": "org.apache.hadoop:hadoop-common:" + hadoopVersion,