You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by fg...@apache.org on 2018/01/23 16:44:17 UTC
[20/32] incubator-sdap-ningester git commit: added other datastores
and their properties
added other datastores and their properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/a52c618e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/a52c618e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/a52c618e
Branch: refs/heads/master
Commit: a52c618eec170634f4b631ac87c2d5afb13de77e
Parents: 53af516
Author: Frank Greguska <fr...@jpl.nasa.gov>
Authored: Mon Jan 8 18:05:39 2018 -0800
Committer: Frank Greguska <fr...@jpl.nasa.gov>
Committed: Mon Jan 8 18:05:39 2018 -0800
----------------------------------------------------------------------
build.gradle | 9 +
.../configuration/DatasourceConfig.java | 167 ++++++++++++++++++-
.../properties/AddDatasetName.java | 24 ---
.../properties/AddDayOfYearAttribute.java | 25 ---
.../properties/AddTimeFromGranuleName.java | 33 ----
.../properties/ApplicationProperties.java | 3 +
.../properties/DatasourceProperties.java | 47 ++++++
.../properties/GenerateTileId.java | 24 ---
.../properties/PythonChainProcessor.java | 42 -----
.../properties/PythonProcessorModule.java | 27 ---
.../properties/SliceFileByDimension.java | 44 -----
.../properties/SliceFileByTilesDesired.java | 44 -----
.../properties/SliceFileByDimension.java | 44 +++++
.../properties/SliceFileByTilesDesired.java | 44 +++++
.../processors/PythonChainProcessor.java | 2 +-
.../processors/properties/AddDatasetName.java | 24 +++
.../properties/AddDayOfYearAttribute.java | 25 +++
.../properties/AddTimeFromGranuleName.java | 33 ++++
.../processors/properties/GenerateTileId.java | 24 +++
.../properties/PythonChainProcessor.java | 42 +++++
.../properties/PythonProcessorModule.java | 27 +++
.../ningester/writer/DataStoreException.java | 28 ++++
.../jpl/nexus/ningester/writer/DynamoStore.java | 54 ++++++
.../jpl/nexus/ningester/writer/S3Store.java | 62 +++++++
.../jpl/nexus/ningester/writer/SolrStore.java | 3 +-
.../writer/properties/CassandraStore.java | 42 +++++
.../writer/properties/DynamoStore.java | 44 +++++
.../ningester/writer/properties/S3Store.java | 34 ++++
.../ningester/writer/properties/SolrStore.java | 44 +++++
src/main/resources/application.properties | 0
src/main/resources/application.yml | 5 +
31 files changed, 797 insertions(+), 273 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index ebabedf..7ceb035 100644
--- a/build.gradle
+++ b/build.gradle
@@ -59,6 +59,12 @@ sourceSets{
}
}
+dependencyManagement {
+ imports {
+ mavenBom 'com.amazonaws:aws-java-sdk-bom:1.10.77+'
+ }
+}
+
dependencies {
optional("org.springframework.boot:spring-boot-configuration-processor")
@@ -74,6 +80,9 @@ dependencies {
compile("edu.ucar:cdm:${netcdfJavaVersion}")
compile("com.google.guava:guava")
+ compile("com.amazonaws:aws-java-sdk-s3")
+ compile("com.amazonaws:aws-java-sdk-dynamodb")
+
testCompile('org.springframework.boot:spring-boot-starter-test')
testCompile('org.springframework.batch:spring-batch-test')
}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java
index 4868673..de4fef2 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java
@@ -1,8 +1,32 @@
package gov.nasa.jpl.nexus.ningester.configuration;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient;
+import com.amazonaws.services.s3.AmazonS3Client;
+import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties;
+import gov.nasa.jpl.nexus.ningester.writer.*;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
+import org.springframework.data.cassandra.config.CassandraClusterFactoryBean;
+import org.springframework.data.cassandra.config.CassandraSessionFactoryBean;
+import org.springframework.data.cassandra.config.SchemaAction;
+import org.springframework.data.cassandra.convert.CassandraConverter;
+import org.springframework.data.cassandra.convert.MappingCassandraConverter;
+import org.springframework.data.cassandra.core.CassandraOperations;
+import org.springframework.data.cassandra.core.CassandraTemplate;
+import org.springframework.data.cassandra.mapping.BasicCassandraMappingContext;
+import org.springframework.data.cassandra.mapping.CassandraMappingContext;
+import org.springframework.data.solr.core.SolrOperations;
+import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
@@ -11,12 +35,139 @@ import javax.sql.DataSource;
@Configuration
public class DatasourceConfig {
- @Bean
- @Profile("embedded")
- public DataSource dataSource(){
- EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder();
- return builder
- .setType(EmbeddedDatabaseType.H2)
- .build();
- }
+ private final DatasourceProperties datasourceProperties;
+
+ @Autowired
+ public DatasourceConfig(DatasourceProperties datasourceProperties) {
+ this.datasourceProperties = datasourceProperties;
+ }
+
+ @Bean
+ @Profile("embedded")
+ public DataSource dataSource() {
+ EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder();
+ return builder
+ .setType(EmbeddedDatabaseType.H2)
+ .build();
+ }
+
+
+ @Configuration
+ @Profile("cassandra")
+ class CassandraConfiguration {
+ @Bean
+ public CassandraClusterFactoryBean cluster() {
+
+ CassandraClusterFactoryBean cluster = new CassandraClusterFactoryBean();
+ cluster.setContactPoints(datasourceProperties.getCassandraStore().getContactPoints());
+ cluster.setPort(datasourceProperties.getCassandraStore().getPort());
+
+ return cluster;
+ }
+
+ @Bean
+ public CassandraMappingContext mappingContext() {
+ return new BasicCassandraMappingContext();
+ }
+
+ @Bean
+ public CassandraConverter converter() {
+ return new MappingCassandraConverter(mappingContext());
+ }
+
+ @Bean
+ public CassandraSessionFactoryBean session(){
+
+ CassandraSessionFactoryBean session = new CassandraSessionFactoryBean();
+ session.setCluster(cluster().getObject());
+ session.setKeyspaceName(datasourceProperties.getCassandraStore().getKeyspace());
+ session.setConverter(converter());
+ session.setSchemaAction(SchemaAction.NONE);
+
+ return session;
+ }
+
+ @Bean
+ public CassandraOperations cassandraTemplate(){
+ return new CassandraTemplate(session().getObject());
+ }
+
+ @Bean
+ public DataStore dataStore(CassandraOperations cassandraTemplate) {
+ return new CassandraStore(cassandraTemplate);
+ }
+ }
+
+ @Configuration
+ @Profile("dynamo")
+ class DynamoConfiguration {
+ @Bean
+ public AmazonDynamoDB dynamoClient() {
+ AmazonDynamoDB dynamoClient = new AmazonDynamoDBClient();
+ dynamoClient.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getDynamoStore().getRegion())));
+ return dynamoClient;
+ }
+
+ @Bean
+ public DataStore dataStore(AmazonDynamoDB dynamoClient) {
+ return new DynamoStore(dynamoClient,
+ datasourceProperties.getDynamoStore().getTableName(),
+ datasourceProperties.getDynamoStore().getPrimaryKey());
+ }
+ }
+
+ @Configuration
+ @Profile("s3")
+ class S3Configuration {
+ @Bean
+ public AmazonS3Client s3client() {
+ AmazonS3Client s3Client = new AmazonS3Client();
+ s3Client.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getS3Store().getRegion())));
+ return s3Client;
+ }
+
+ @Bean
+ public DataStore dataStore(AmazonS3Client s3Client) {
+ return new S3Store(s3Client, datasourceProperties.getS3Store().getBucketName());
+ }
+ }
+
+ @Configuration
+ @Profile("solr-standalone")
+ class SolrConfiguration {
+ @Bean
+ public SolrClient solrClient(){ return new HttpSolrClient(datasourceProperties.getSolrStore().getUrl() + datasourceProperties.getSolrStore().getCollection());}
+
+ @Bean
+ public SolrOperations solrTemplate(SolrClient solrClient) {
+ return new SolrTemplate(solrClient);
+ }
+
+ @Bean
+ public MetadataStore metadataStore(SolrOperations solrTemplate) {
+ return new SolrStore(solrTemplate);
+ }
+ }
+
+ @Configuration
+ @Profile("solr-cloud")
+ class SolrCloudConfiguration {
+ @Bean
+ public SolrClient solrClient(){
+ CloudSolrClient client = new CloudSolrClient(datasourceProperties.getSolrStore().getZkHost());
+ client.setDefaultCollection(datasourceProperties.getSolrStore().getCollection());
+
+ return client;
+ }
+
+ @Bean
+ public SolrOperations solrTemplate(SolrClient solrClient) {
+ return new SolrTemplate(solrClient);
+ }
+
+ @Bean
+ public MetadataStore metadataStore(SolrOperations solrTemplate) {
+ return new SolrStore(solrTemplate);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java
deleted file mode 100644
index 8ff9230..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@ConfigurationProperties
-@Component("addDatasetNameProperties")
-public class AddDatasetName {
-
- private String datasetName;
-
- public String getDatasetName() {
- return datasetName;
- }
-
- public void setDatasetName(String datasetName) {
- this.datasetName = datasetName;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java
deleted file mode 100644
index 7ec9513..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@ConfigurationProperties
-@Component("addDayOfYearAttributeProperties")
-public class AddDayOfYearAttribute {
-
- private String regex;
-
-
- public String getRegex() {
- return regex;
- }
-
- public void setRegex(String regex) {
- this.regex = regex;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java
deleted file mode 100644
index 2995bd4..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@ConfigurationProperties
-@Component("addTimeFromGranuleNameProperties")
-public class AddTimeFromGranuleName {
-
- private String regex;
- private String dateFormat;
-
- public String getRegex() {
- return regex;
- }
-
- public void setRegex(String regex) {
- this.regex = regex;
- }
-
- public String getDateFormat() {
- return dateFormat;
- }
-
- public void setDateFormat(String dateFormat) {
- this.dateFormat = dateFormat;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
index e6ea647..9dace2e 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
@@ -5,6 +5,9 @@
package gov.nasa.jpl.nexus.ningester.configuration.properties;
+import gov.nasa.jpl.nexus.ningester.datatiler.properties.SliceFileByDimension;
+import gov.nasa.jpl.nexus.ningester.datatiler.properties.SliceFileByTilesDesired;
+import gov.nasa.jpl.nexus.ningester.processors.properties.*;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.stereotype.Component;
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java
new file mode 100644
index 0000000..228ec1e
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java
@@ -0,0 +1,47 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.configuration.properties;
+
+import gov.nasa.jpl.nexus.ningester.writer.properties.CassandraStore;
+import gov.nasa.jpl.nexus.ningester.writer.properties.DynamoStore;
+import gov.nasa.jpl.nexus.ningester.writer.properties.S3Store;
+import gov.nasa.jpl.nexus.ningester.writer.properties.SolrStore;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties("datasource")
+@Component
+public class DatasourceProperties {
+
+ @NestedConfigurationProperty
+ private final CassandraStore cassandraStore = new CassandraStore();
+
+ @NestedConfigurationProperty
+ private final DynamoStore dynamoStore = new DynamoStore();
+
+ @NestedConfigurationProperty
+ private final S3Store s3Store = new S3Store();
+
+ @NestedConfigurationProperty
+ private final SolrStore solrStore = new SolrStore();
+
+ public CassandraStore getCassandraStore() {
+ return cassandraStore;
+ }
+
+ public DynamoStore getDynamoStore() {
+ return dynamoStore;
+ }
+
+ public S3Store getS3Store() {
+ return s3Store;
+ }
+
+ public SolrStore getSolrStore() {
+ return solrStore;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java
deleted file mode 100644
index 05caf6f..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-@ConfigurationProperties
-@Component("generateTileIdProperties")
-public class GenerateTileId {
-
- private String salt = "";
-
- public String getSalt() {
- return salt;
- }
-
- public void setSalt(String salt) {
- this.salt = salt;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java
deleted file mode 100644
index ff38025..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2017 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-
-@ConfigurationProperties
-@Component("pythonChainProcessorProperties")
-public class PythonChainProcessor {
- private URL baseUrl;
- private String uriPath;
-
- private List<PythonProcessorModule> processorList = new ArrayList<>();
-
- public String getUriPath() {
- return uriPath;
- }
-
- public void setUriPath(String uriPath) {
- this.uriPath = uriPath;
- }
-
- public List<PythonProcessorModule> getProcessorList() {
- return processorList;
- }
-
- public URL getBaseUrl() {
- return baseUrl;
- }
-
- public void setBaseUrl(URL baseUrl) {
- this.baseUrl = baseUrl;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java
deleted file mode 100644
index 287e449..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2017 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class PythonProcessorModule {
-
- private String name;
- private Map<String, String> config = new HashMap<>();
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public Map<String, String> getConfig() {
- return config;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java
deleted file mode 100644
index 4be4187..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-import java.util.List;
-
-@ConfigurationProperties
-@Component("sliceFileByDimensionProperties")
-public class SliceFileByDimension {
-
- private String sliceByDimension;
- private List<String> dimensions;
- private String dimensionNamePrefix = "";
-
- public String getSliceByDimension() {
- return sliceByDimension;
- }
-
- public void setSliceByDimension(String sliceByDimension) {
- this.sliceByDimension = sliceByDimension;
- }
-
- public List<String> getDimensions() {
- return dimensions;
- }
-
- public void setDimensions(List<String> dimensions) {
- this.dimensions = dimensions;
- }
-
- public String getDimensionNamePrefix() {
- return dimensionNamePrefix;
- }
-
- public void setDimensionNamePrefix(String dimensionNamePrefix) {
- this.dimensionNamePrefix = dimensionNamePrefix;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java
deleted file mode 100644
index aa4d542..0000000
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.configuration.properties;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.stereotype.Component;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@ConfigurationProperties
-@Component("sliceFileByTilesDesiredProperties")
-public class SliceFileByTilesDesired {
- private Integer tilesDesired;
- private List<String> dimensions = new ArrayList<>();
- private String timeDimension;
-
- public Integer getTilesDesired() {
- return tilesDesired;
- }
-
- public void setTilesDesired(Integer tilesDesired) {
- this.tilesDesired = tilesDesired;
- }
-
- public List<String> getDimensions() {
- return dimensions;
- }
-
- public void setDimensions(List<String> dimensions) {
- this.dimensions = dimensions;
- }
-
- public String getTimeDimension() {
- return timeDimension;
- }
-
- public void setTimeDimension(String timeDimension) {
- this.timeDimension = timeDimension;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java
new file mode 100644
index 0000000..0482e55
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java
@@ -0,0 +1,44 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.datatiler.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+@ConfigurationProperties
+@Component("sliceFileByDimensionProperties")
+public class SliceFileByDimension {
+
+ private String sliceByDimension;
+ private List<String> dimensions;
+ private String dimensionNamePrefix = "";
+
+ public String getSliceByDimension() {
+ return sliceByDimension;
+ }
+
+ public void setSliceByDimension(String sliceByDimension) {
+ this.sliceByDimension = sliceByDimension;
+ }
+
+ public List<String> getDimensions() {
+ return dimensions;
+ }
+
+ public void setDimensions(List<String> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ public String getDimensionNamePrefix() {
+ return dimensionNamePrefix;
+ }
+
+ public void setDimensionNamePrefix(String dimensionNamePrefix) {
+ this.dimensionNamePrefix = dimensionNamePrefix;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java
new file mode 100644
index 0000000..cc7933b
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java
@@ -0,0 +1,44 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.datatiler.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@ConfigurationProperties
+@Component("sliceFileByTilesDesiredProperties")
+public class SliceFileByTilesDesired {
+ private Integer tilesDesired;
+ private List<String> dimensions = new ArrayList<>();
+ private String timeDimension;
+
+ public Integer getTilesDesired() {
+ return tilesDesired;
+ }
+
+ public void setTilesDesired(Integer tilesDesired) {
+ this.tilesDesired = tilesDesired;
+ }
+
+ public List<String> getDimensions() {
+ return dimensions;
+ }
+
+ public void setDimensions(List<String> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ public String getTimeDimension() {
+ return timeDimension;
+ }
+
+ public void setTimeDimension(String timeDimension) {
+ this.timeDimension = timeDimension;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java
index 55eb051..0488562 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java
@@ -9,7 +9,7 @@ package gov.nasa.jpl.nexus.ningester.processors;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
-import gov.nasa.jpl.nexus.ningester.configuration.properties.PythonProcessorModule;
+import gov.nasa.jpl.nexus.ningester.processors.properties.PythonProcessorModule;
import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpEntity;
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java
new file mode 100644
index 0000000..7bd9af0
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java
@@ -0,0 +1,24 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.processors.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties
+@Component("addDatasetNameProperties")
+public class AddDatasetName {
+
+ private String datasetName;
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ public void setDatasetName(String datasetName) {
+ this.datasetName = datasetName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java
new file mode 100644
index 0000000..33d8b24
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java
@@ -0,0 +1,25 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.processors.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties
+@Component("addDayOfYearAttributeProperties")
+public class AddDayOfYearAttribute {
+
+ private String regex;
+
+
+ public String getRegex() {
+ return regex;
+ }
+
+ public void setRegex(String regex) {
+ this.regex = regex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java
new file mode 100644
index 0000000..1042d6a
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java
@@ -0,0 +1,33 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.processors.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties
+@Component("addTimeFromGranuleNameProperties")
+public class AddTimeFromGranuleName {
+
+ private String regex;
+ private String dateFormat;
+
+ public String getRegex() {
+ return regex;
+ }
+
+ public void setRegex(String regex) {
+ this.regex = regex;
+ }
+
+ public String getDateFormat() {
+ return dateFormat;
+ }
+
+ public void setDateFormat(String dateFormat) {
+ this.dateFormat = dateFormat;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java
new file mode 100644
index 0000000..60cf75c
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java
@@ -0,0 +1,24 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.processors.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties
+@Component("generateTileIdProperties")
+public class GenerateTileId {
+
+ private String salt = "";
+
+ public String getSalt() {
+ return salt;
+ }
+
+ public void setSalt(String salt) {
+ this.salt = salt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java
new file mode 100644
index 0000000..ac86d99
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java
@@ -0,0 +1,42 @@
+/*****************************************************************************
+ * Copyright (c) 2017 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.processors.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+@ConfigurationProperties
+@Component("pythonChainProcessorProperties")
+public class PythonChainProcessor {
+ private URL baseUrl;
+ private String uriPath;
+
+ private List<PythonProcessorModule> processorList = new ArrayList<>();
+
+ public String getUriPath() {
+ return uriPath;
+ }
+
+ public void setUriPath(String uriPath) {
+ this.uriPath = uriPath;
+ }
+
+ public List<PythonProcessorModule> getProcessorList() {
+ return processorList;
+ }
+
+ public URL getBaseUrl() {
+ return baseUrl;
+ }
+
+ public void setBaseUrl(URL baseUrl) {
+ this.baseUrl = baseUrl;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java
new file mode 100644
index 0000000..e48f0d5
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java
@@ -0,0 +1,27 @@
+/*****************************************************************************
+ * Copyright (c) 2017 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.processors.properties;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PythonProcessorModule {
+
+ private String name;
+ private Map<String, String> config = new HashMap<>();
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Map<String, String> getConfig() {
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java
new file mode 100644
index 0000000..9ba838e
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java
@@ -0,0 +1,28 @@
+/*
+ *****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+package gov.nasa.jpl.nexus.ningester.writer;
+
+public class DataStoreException extends RuntimeException {
+
+ public DataStoreException() {
+ }
+
+ public DataStoreException(String message) {
+ super(message);
+ }
+
+ public DataStoreException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public DataStoreException(Throwable cause) {
+ super(cause);
+ }
+
+ public DataStoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java
new file mode 100644
index 0000000..4c13a14
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java
@@ -0,0 +1,54 @@
+/*
+ *****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+package gov.nasa.jpl.nexus.ningester.writer;
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
+import com.amazonaws.services.dynamodbv2.document.DynamoDB;
+import com.amazonaws.services.dynamodbv2.document.Item;
+import com.amazonaws.services.dynamodbv2.document.Table;
+import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent;
+
+import java.util.Collection;
+
+/**
+ * Created by djsilvan on 6/26/17.
+ */
+public class DynamoStore implements DataStore {
+
+ private DynamoDB dynamoDB;
+ private String tableName;
+ private String primaryKey;
+
+ public DynamoStore(AmazonDynamoDB dynamoClient, String tableName, String primaryKey) {
+ dynamoDB = new DynamoDB(dynamoClient);
+ this.tableName = tableName;
+ this.primaryKey = primaryKey;
+ }
+
+ public void saveData(Collection<NexusContent.NexusTile> nexusTiles) {
+
+ Table table = dynamoDB.getTable(tableName);
+
+ for (NexusContent.NexusTile tile : nexusTiles) {
+ String tileId = getTileId(tile);
+ byte[] tileData = getTileData(tile);
+
+ try {
+ table.putItem(new Item().withPrimaryKey(primaryKey, tileId).withBinary("data", tileData));
+ } catch (Exception e) {
+ throw new DataStoreException("Unable to add item: " + tileId, e);
+ }
+ }
+ }
+
+ private String getTileId(NexusContent.NexusTile tile) {
+ return tile.getTile().getTileId();
+ }
+
+ private byte[] getTileData(NexusContent.NexusTile tile) {
+ return tile.getTile().toByteArray();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java
new file mode 100644
index 0000000..a3f0e22
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java
@@ -0,0 +1,62 @@
+/*****************************************************************************
+ * Copyright (c) 2017 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+package gov.nasa.jpl.nexus.ningester.writer;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent.NexusTile;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.Collection;
+
+/**
+ * Created by djsilvan on 6/26/17.
+ */
+public class S3Store implements DataStore {
+
+ private AmazonS3 s3;
+ private String bucketName;
+
+ public S3Store(AmazonS3Client s3client, String bucketName) {
+ s3 = s3client;
+ this.bucketName = bucketName;
+ }
+
+ public void saveData(Collection<NexusTile> nexusTiles) {
+
+ for (NexusTile tile : nexusTiles) {
+ String tileId = getTileId(tile);
+ byte[] tileData = getTileData(tile);
+ Long contentLength = (long) tileData.length;
+ InputStream stream = new ByteArrayInputStream(tileData);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(contentLength);
+
+ try {
+ s3.putObject(new PutObjectRequest(bucketName, tileId, stream, meta));
+ } catch (AmazonServiceException ase) {
+ throw new DataStoreException("Caught an AmazonServiceException, which means your request made it "
+ + "to Amazon S3, but was rejected with an error response for some reason.", ase);
+ } catch (AmazonClientException ace) {
+ throw new DataStoreException("Caught an AmazonClientException, which means the client encountered "
+ + "a serious internal problem while trying to communicate with S3, "
+ + "such as not being able to access the network.", ace);
+ }
+ }
+ }
+
+ private String getTileId(NexusTile tile) {
+ return tile.getTile().getTileId();
+ }
+
+ private byte[] getTileData(NexusTile tile) {
+ return tile.getTile().toByteArray();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java
index 71f4ef0..1706b77 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java
@@ -20,7 +20,7 @@ import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
-public class SolrStore {
+public class SolrStore implements MetadataStore {
private Environment environment;
private SolrOperations solr;
@@ -44,6 +44,7 @@ public class SolrStore {
this.environment = environment;
}
+ @Override
public void saveMetadata(Collection<NexusContent.NexusTile> nexusTiles) {
List<SolrInputDocument> solrdocs = nexusTiles.stream()
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java
new file mode 100644
index 0000000..c6da68e
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java
@@ -0,0 +1,42 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.writer.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties("cassandra")
+@Component("cassandraStoreProperties")
+public class CassandraStore {
+
+ private String contactPoints;
+ private String keyspace;
+ private Integer port;
+
+ public String getContactPoints() {
+ return contactPoints;
+ }
+
+ public void setContactPoints(String contactPoints) {
+ this.contactPoints = contactPoints;
+ }
+
+ public String getKeyspace() {
+ return keyspace;
+ }
+
+ public void setKeyspace(String keyspace) {
+ this.keyspace = keyspace;
+ }
+
+ public Integer getPort() {
+ return port;
+ }
+
+ public void setPort(Integer port) {
+ this.port = port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java
new file mode 100644
index 0000000..455396e
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java
@@ -0,0 +1,44 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.writer.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties("dynamo")
+@Component("dynamoStoreProperties")
+public class DynamoStore {
+
+ private String tableName;
+
+ private String region;
+
+ private String primaryKey = "tile_id";
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getRegion() {
+ return region;
+ }
+
+ public void setRegion(String region) {
+ this.region = region;
+ }
+
+ public String getPrimaryKey() {
+ return primaryKey;
+ }
+
+ public void setPrimaryKey(String primaryKey) {
+ this.primaryKey = primaryKey;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java
new file mode 100644
index 0000000..367e844
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java
@@ -0,0 +1,34 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.writer.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties("s3")
+@Component("s3StoreProperties")
+public class S3Store {
+
+ private String bucketName;
+
+ private String region;
+
+ public String getRegion() {
+ return region;
+ }
+
+ public void setRegion(String region) {
+ this.region = region;
+ }
+
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ public void setBucketName(String bucketName) {
+ this.bucketName = bucketName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java
new file mode 100644
index 0000000..3dbb41d
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java
@@ -0,0 +1,44 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.writer.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@ConfigurationProperties("solr")
+@Component("solrStoreProperties")
+public class SolrStore {
+
+
+ private String url;
+ private String zkHost;
+ private String collection;
+
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+
+ public String getZkHost() {
+ return zkHost;
+ }
+
+ public void setZkHost(String zkHost) {
+ this.zkHost = zkHost;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/resources/application.properties
----------------------------------------------------------------------
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
new file mode 100644
index 0000000..b536524
--- /dev/null
+++ b/src/main/resources/application.yml
@@ -0,0 +1,5 @@
+spring:
+ autoconfigure:
+ exclude:
+ - org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration
+ - org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration
\ No newline at end of file