You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by fg...@apache.org on 2018/01/23 16:44:14 UTC
[17/32] incubator-sdap-ningester git commit: wired in the
slicebydimension properties and added smap testjob
wired in the slicebydimension properties and added smap testjob
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/4f27864e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/4f27864e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/4f27864e
Branch: refs/heads/master
Commit: 4f27864e8b0350b552688eed7c95b52899a723f7
Parents: 39ea2ef
Author: Frank Greguska <fr...@jpl.nasa.gov>
Authored: Thu Jan 4 15:40:08 2018 -0800
Committer: Frank Greguska <fr...@jpl.nasa.gov>
Committed: Thu Jan 4 15:40:08 2018 -0800
----------------------------------------------------------------------
.../ningester/configuration/AppConfig.java | 16 ++++--
.../properties/ApplicationProperties.java | 7 +++
.../properties/SliceFileByDimension.java | 44 +++++++++++++++
.../datatiler/SliceFileByDimension.java | 9 +++-
.../ningester/processors/GenerateTileId.java | 15 +++++-
.../processors/TestGenerateTileId.java | 34 +++++-------
.../nexus/ningester/testjobs/SmapJobTest.java | 54 +++++++++++++++++++
src/test/resources/testjobs/SmapJobTest.yml | 57 ++++++++++++++++++++
8 files changed, 208 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java
index ff3c1e0..3354c40 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java
@@ -7,6 +7,7 @@ package gov.nasa.jpl.nexus.ningester.configuration;
import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties;
import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer;
+import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByDimension;
import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired;
import gov.nasa.jpl.nexus.ningester.http.NexusTileConverter;
import gov.nasa.jpl.nexus.ningester.processors.*;
@@ -15,7 +16,6 @@ import org.springframework.batch.core.configuration.annotation.JobScope;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
@@ -27,7 +27,6 @@ import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.DefaultUriTemplateHandler;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -45,7 +44,7 @@ public class AppConfig {
@Bean
@ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByTilesDesired")
@Qualifier("fileSlicer")
- protected FileSlicer sliceFileByTilesDesired(){
+ protected FileSlicer sliceFileByTilesDesired() {
SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired();
fileSlicer.setDimensions(applicationProperties.getSliceFileByTilesDesired().getDimensions());
fileSlicer.setTilesDesired(applicationProperties.getSliceFileByTilesDesired().getTilesDesired());
@@ -54,6 +53,17 @@ public class AppConfig {
}
@Bean
+ @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByDimension")
+ @Qualifier("fileSlicer")
+ protected FileSlicer sliceFileByDimension() {
+ SliceFileByDimension fileSlicer = new SliceFileByDimension();
+ fileSlicer.setDimensions(applicationProperties.getSliceFileByDimension().getDimensions());
+ fileSlicer.setSliceByDimension(applicationProperties.getSliceFileByDimension().getSliceByDimension());
+ fileSlicer.setDimensionNamePrefix(applicationProperties.getSliceFileByDimension().getDimensionNamePrefix());
+ return fileSlicer;
+ }
+
+ @Bean
protected HttpMessageConverter nexusTileConverter() {
NexusTileConverter converter = new NexusTileConverter();
converter.setSupportedMediaTypes(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM));
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
index f94e53c..e6ea647 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java
@@ -19,6 +19,9 @@ public class ApplicationProperties {
private String tileSlicer;
@NestedConfigurationProperty
+ private final SliceFileByDimension sliceFileByDimension = new SliceFileByDimension();
+
+ @NestedConfigurationProperty
private final SliceFileByTilesDesired sliceFileByTilesDesired = new SliceFileByTilesDesired();
private List<String> tileProcessors = new ArrayList<>();
@@ -73,4 +76,8 @@ public class ApplicationProperties {
public SliceFileByTilesDesired getSliceFileByTilesDesired() {
return sliceFileByTilesDesired;
}
+
+ public SliceFileByDimension getSliceFileByDimension() {
+ return sliceFileByDimension;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java
new file mode 100644
index 0000000..4be4187
--- /dev/null
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java
@@ -0,0 +1,44 @@
+/*****************************************************************************
+ * Copyright (c) 2018 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.configuration.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+@ConfigurationProperties
+@Component("sliceFileByDimensionProperties")
+public class SliceFileByDimension {
+
+ private String sliceByDimension;
+ private List<String> dimensions;
+ private String dimensionNamePrefix = "";
+
+ public String getSliceByDimension() {
+ return sliceByDimension;
+ }
+
+ public void setSliceByDimension(String sliceByDimension) {
+ this.sliceByDimension = sliceByDimension;
+ }
+
+ public List<String> getDimensions() {
+ return dimensions;
+ }
+
+ public void setDimensions(List<String> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ public String getDimensionNamePrefix() {
+ return dimensionNamePrefix;
+ }
+
+ public void setDimensionNamePrefix(String dimensionNamePrefix) {
+ this.dimensionNamePrefix = dimensionNamePrefix;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java
index 003edaa..0a5b594 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java
@@ -5,6 +5,7 @@
* ****************************************************************************/
package gov.nasa.jpl.nexus.ningester.datatiler;
+import com.google.common.base.Strings;
import com.google.common.collect.Sets;
import ucar.nc2.Dimension;
import ucar.nc2.Variable;
@@ -16,7 +17,7 @@ import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-class SliceFileByDimension implements FileSlicer {
+public class SliceFileByDimension implements FileSlicer {
private String sliceByDimension;
private List<String> dimensions;
@@ -47,6 +48,12 @@ class SliceFileByDimension implements FileSlicer {
}
List<String> indexedDimensionSlicing(File inputfile) throws IOException {
+
+ // This is sort of a hack to help the python netcdf library. When you try to get dimensions by name and they are unnamed, the
+ // python library uses 'phony_dim_' then the index of the dimension as the dimension name. Weird, I know.
+ if(Strings.isNullOrEmpty(this.dimensionNamePrefix)){
+ this.dimensionNamePrefix = "phony_dim_";
+ }
Map<String, Integer> dimensionNameToLength;
try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) {
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java
index fba1193..6d00a99 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java
@@ -6,8 +6,13 @@
package gov.nasa.jpl.nexus.ningester.processors;
+import com.google.common.io.Files;
import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.UUID;
public class GenerateTileId {
@@ -18,11 +23,17 @@ public class GenerateTileId {
this.salt = salt;
}
- public NexusContent.NexusTile addTileId(NexusContent.NexusTile inputTile) {
+ public NexusContent.NexusTile addTileId(NexusContent.NexusTile inputTile){
NexusContent.NexusTile.Builder outTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(inputTile);
String granuleFileName = inputTile.getSummary().getGranule();
- String granuleName = granuleFileName.substring(0, granuleFileName.length() - 3);
+ Path granulePath = null;
+ try {
+ granulePath = Paths.get(new URI(granuleFileName));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ String granuleName = Files.getNameWithoutExtension(granulePath.getFileName().toString());
String spec = inputTile.getSummary().getSectionSpec();
String tileId = UUID.nameUUIDFromBytes((granuleName + spec + salt).getBytes()).toString();
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java
----------------------------------------------------------------------
diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java
index d000479..ea18c3a 100644
--- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java
+++ b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java
@@ -17,24 +17,19 @@ public class TestGenerateTileId {
@Test
public void testGenerateId() {
- String granuleFileName = "19960421120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc";
+ String granuleFileName = "file:/path/to/some/file/19960421120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc";
String sectionSpec = "time:0:1,lat:140:160,lon:640:680";
String expectedId = "c031a9c4-9e1d-32e9-9d5c-d2497ce74920";
- NexusContent.NexusTile inputTile = NexusContent.NexusTile.newBuilder()
- .setSummary(
- NexusContent.TileSummary.newBuilder()
- .setGranule(granuleFileName)
- .setSectionSpec(sectionSpec)
- .build()
- )
- .setTile(NexusContent.TileData.newBuilder())
- .build();
+ NexusContent.NexusTile.Builder inputTileBuilder = NexusContent.NexusTile.newBuilder();
+ inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName);
+ inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec);
+ inputTileBuilder.setTile(NexusContent.TileData.newBuilder());
GenerateTileId processor = new GenerateTileId();
- NexusContent.NexusTile result = processor.addTileId(inputTile);
+ NexusContent.NexusTile result = processor.addTileId(inputTileBuilder.build());
assertThat(result.getSummary().getTileId(), is(expectedId));
assertThat(result.getTile().getTileId(), is(expectedId));
@@ -44,26 +39,21 @@ public class TestGenerateTileId {
@Test
public void testGenerateIdWithSalt() {
- String granuleFileName = "CCMP_Wind_Analysis_19990928_V02.0_L3.0_RSS.nc";
+ String granuleFileName = "file:/path/to/some/file/CCMP_Wind_Analysis_19990928_V02.0_L3.0_RSS.nc";
String sectionSpec = "time:3:4,longitude:174:261,latitude:152:190";
String salt = "wind_u";
String expectedId = "48da50ef-e92c-3562-89f9-470561a06482";
- NexusContent.NexusTile inputTile = NexusContent.NexusTile.newBuilder()
- .setSummary(
- NexusContent.TileSummary.newBuilder()
- .setGranule(granuleFileName)
- .setSectionSpec(sectionSpec)
- .build()
- )
- .setTile(NexusContent.TileData.newBuilder())
- .build();
+ NexusContent.NexusTile.Builder inputTileBuilder = NexusContent.NexusTile.newBuilder();
+ inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName);
+ inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec);
+ inputTileBuilder.setTile(NexusContent.TileData.newBuilder());
GenerateTileId processor = new GenerateTileId();
processor.setSalt(salt);
- NexusContent.NexusTile result = processor.addTileId(inputTile);
+ NexusContent.NexusTile result = processor.addTileId(inputTileBuilder.build());
assertThat(result.getSummary().getTileId(), is(expectedId));
assertThat(result.getTile().getTileId(), is(expectedId));
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java
new file mode 100644
index 0000000..d99dc2b
--- /dev/null
+++ b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java
@@ -0,0 +1,54 @@
+/*****************************************************************************
+ * Copyright (c) 2017 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+
+package gov.nasa.jpl.nexus.ningester.testjobs;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.test.JobLauncherTestUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+@TestPropertySource(properties = { "spring.config.location = classpath:testjobs/SmapJobTest.yml" })
+@ActiveProfiles({"test"})
+public class SmapJobTest {
+
+ @TestConfiguration
+ static class NingesterApplicationTestsConfig {
+
+ @Bean
+ JobLauncherTestUtils jobLauncherTestUtils() {
+ return new JobLauncherTestUtils();
+ }
+
+ }
+
+ @Autowired
+ JobLauncherTestUtils jobLauncherTestUtils;
+
+
+ @Test
+ public void testJobCompletes() throws Exception {
+
+ JobParameters jobParameters = new JobParametersBuilder()
+ .addString("granule", "classpath:granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5")
+ .toJobParameters();
+
+ JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters);
+
+ Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/test/resources/testjobs/SmapJobTest.yml
----------------------------------------------------------------------
diff --git a/src/test/resources/testjobs/SmapJobTest.yml b/src/test/resources/testjobs/SmapJobTest.yml
new file mode 100644
index 0000000..e8fda0d
--- /dev/null
+++ b/src/test/resources/testjobs/SmapJobTest.yml
@@ -0,0 +1,57 @@
+spring:
+ profiles: test
+
+---
+# Tile Slicer Config
+ningester:
+ tile_slicer: sliceFileByDimension
+ sliceFileByDimension:
+ sliceByDimension: 1
+ dimensionNamePrefix: phony_dim_
+ dimensions:
+ - 0
+ - 1
+
+---
+# Tile processors configuration
+ningester:
+ tile_processors:
+ - pythonChainProcessor
+ - generateTileId
+ - addDatasetName
+ pythonChainProcessor:
+ enabled:
+ base_url: http://127.0.0.1:5000/
+ uri_path: processorchain
+ processor_list:
+ -
+ name: SwathReadingProcessor
+ config:
+ latitude: lat
+ longitude: lon
+ time: row_time
+ variable_to_read: smap_sss
+ glblattr_day: REV_START_TIME
+ glblattr_day_format: '%Y-%jT%H:%M:%S.%f'
+ -
+ name: EmptyTileFilter
+ -
+ name: TileSummarizingProcessor
+ generateTileId:
+ enabled:
+ addDatasetName:
+ enabled:
+ datasetName: SMAP_L2B_SSS
+---
+# Tile writer configuration
+ningester:
+ tile_writer:
+ data_store: cassandraStore
+ metadata_store: solrStore
+ cassandraStore:
+ cassandraContactPoints: 127.0.0.1
+ cassandraKeyspace: nexustiles
+ cassandraPort: 9042
+ solrStore:
+ solrUrl: http://127.0.0.1:8983
+ solrCollection: nexustiles
\ No newline at end of file