You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by fg...@apache.org on 2018/01/23 16:44:04 UTC
[07/32] incubator-sdap-ningester git commit: updated reader and tests
updated reader and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/e5e4ef73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/e5e4ef73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/e5e4ef73
Branch: refs/heads/master
Commit: e5e4ef73e06fcd097750238fd9dda95480216158
Parents: a6e6e2a
Author: Frank Greguska <fr...@jpl.nasa.gov>
Authored: Tue Nov 7 15:00:04 2017 -0800
Committer: Frank Greguska <fr...@jpl.nasa.gov>
Committed: Tue Nov 7 15:00:04 2017 -0800
----------------------------------------------------------------------
.gitignore | 2 +
.../ningester/datatiler/NetCDFItemReader.java | 41 ++++---
.../datatiler/NetCDFItemReaderTest.java | 107 +++++--------------
3 files changed, 59 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/e5e4ef73/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index c9f7010..72001d3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
/build/
!gradle/wrapper/gradle-wrapper.jar
+/out
+
### STS ###
.apt_generated
.classpath
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/e5e4ef73/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java
index 45e68f9..35dfef1 100644
--- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java
+++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java
@@ -5,19 +5,24 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.*;
import org.springframework.beans.factory.annotation.Autowired;
+import ucar.ma2.Array;
+import ucar.nc2.Variable;
import ucar.nc2.dataset.NetcdfDataset;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, ItemStream {
+public class NetCDFItemReader implements ItemReader<String>, ItemStream {
private static final Logger log = LoggerFactory.getLogger(NetCDFItemReader.class);
- private static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index";
+ static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index";
private List<String> tileSpecList;
private Integer currentTileSpecIndex;
@@ -32,7 +37,7 @@ public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, Ite
* @param fileSlicer Object responsible for slicing the NetCDF file into tiles.
*/
@Autowired
- public NetCDFItemReader(FileSlicer fileSlicer){
+ public NetCDFItemReader(FileSlicer fileSlicer) {
this.fileSlicer = fileSlicer;
}
@@ -42,13 +47,25 @@ public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, Ite
}
@Override
- public NexusContent.NexusTile read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
+ public String read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
String currentSpec = this.tileSpecList.get(this.currentTileSpecIndex);
- currentSpec.split("");
- this.ds.getVariables().get(0).read(this.tileSpecList.get(this.currentTileSpecIndex++));
-
- return null;
+ Map<String, String> dimensionToSpec = Arrays.stream(currentSpec.split(","))
+ .collect(Collectors.toMap(
+ dimension -> dimension.split(":")[0],
+ dimension -> dimension.substring(dimension.indexOf(":") + 1, dimension.length())));
+
+ Variable varToRead = this.ds.getVariables().get(0);
+ String spec = varToRead.getDimensions().stream()
+ .map(dimension -> dimensionToSpec.get(dimension.getShortName()))
+ .filter(Objects::nonNull)
+ .collect(Collectors.joining(","));
+
+ spec = ":," + spec;
+ Array data = this.ds.getVariables().get(0).read(spec);
+
+ this.currentTileSpecIndex++;
+ return data.toString();
}
@Override
@@ -56,16 +73,16 @@ public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, Ite
//Every time we open the file we generate the tile specs according to the given file slicer
this.tileSpecList = fileSlicer.generateSlices(this.netCDFFile);
- log.debug("Generated tile specifications for {}\n{}", this.netCDFFile.getName(),
+ log.debug("Generated tile specifications for {}\nINDEX\tTILE SPECIFICATION\n{}", this.netCDFFile.getName(),
IntStream.range(0, this.tileSpecList.size())
- .mapToObj(i -> i + ": " + this.tileSpecList.get(i))
+ .mapToObj(i -> i + "\t" + this.tileSpecList.get(i))
.collect(Collectors.joining("\n")));
- if(executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) {
+ if (!executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) {
//Start at index 0
this.currentTileSpecIndex = 0;
executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex);
- }else{
+ } else {
//Start at index location from context
this.currentTileSpecIndex = executionContext.getInt(CURRENT_TILE_SPEC_INDEX_KEY);
}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/e5e4ef73/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java
index 57ce44c..bd0c93e 100644
--- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java
+++ b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java
@@ -1,98 +1,47 @@
package gov.nasa.jpl.nexus.ningester.datatiler;
-import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.batch.item.*;
-import org.springframework.beans.factory.annotation.Autowired;
-import ucar.nc2.dataset.NetcdfDataset;
+import org.junit.Test;
+import org.springframework.batch.item.ExecutionContext;
+import org.springframework.core.io.ClassPathResource;
-import java.io.File;
import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
+import java.util.Arrays;
-public class NetCDFItemReaderTest implements ItemReader<NexusContent.NexusTile>, ItemStream {
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
- private static final Logger log = LoggerFactory.getLogger(NetCDFItemReaderTest.class);
+public class NetCDFItemReaderTest {
- private static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index";
+ @Test
+ public void testOpen() throws IOException {
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+ slicer.setTilesDesired(5184);
+ slicer.setDimensions(Arrays.asList("lat", "lon"));
- private List<String> tileSpecList;
- private Integer currentTileSpecIndex;
+ NetCDFItemReader reader = new NetCDFItemReader(slicer);
+ reader.setNetCDFFile(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc").getFile());
- private File netCDFFile;
- private NetcdfDataset ds;
- private FileSlicer fileSlicer;
+ ExecutionContext context = new ExecutionContext();
+ reader.open(context);
- /**
- * Constructor
- *
- * @param fileSlicer Object responsible for slicing the NetCDF file into tiles.
- */
- @Autowired
- public NetCDFItemReaderTest(FileSlicer fileSlicer){
- this.fileSlicer = fileSlicer;
+ assertTrue(context.containsKey(NetCDFItemReader.CURRENT_TILE_SPEC_INDEX_KEY));
}
- @Autowired
- public void setNetCDFFile(File netCDFFile) {
- this.netCDFFile = netCDFFile;
- }
-
- @Override
- public NexusContent.NexusTile read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
-
- String currentSpec = this.tileSpecList.get(this.currentTileSpecIndex);
- currentSpec.split("");
- this.ds.getVariables().get(0).read(this.tileSpecList.get(this.currentTileSpecIndex++));
-
- return null;
- }
-
- @Override
- public void open(ExecutionContext executionContext) throws ItemStreamException {
+ @Test
+ public void testRead() throws Exception {
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+ slicer.setTilesDesired(5184);
+ slicer.setDimensions(Arrays.asList("lat", "lon"));
- //Every time we open the file we generate the tile specs according to the given file slicer
- this.tileSpecList = fileSlicer.generateSlices(this.netCDFFile);
- log.debug("Generated tile specifications for {}\n{}", this.netCDFFile.getName(),
- IntStream.range(0, this.tileSpecList.size())
- .mapToObj(i -> i + ": " + this.tileSpecList.get(i))
- .collect(Collectors.joining("\n")));
+ NetCDFItemReader reader = new NetCDFItemReader(slicer);
+ reader.setNetCDFFile(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc").getFile());
- if(executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) {
- //Start at index 0
- this.currentTileSpecIndex = 0;
- executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex);
- }else{
- //Start at index location from context
- this.currentTileSpecIndex = executionContext.getInt(CURRENT_TILE_SPEC_INDEX_KEY);
- }
-
- //Open the resource
- try {
- this.ds = NetcdfDataset.openDataset(netCDFFile.getAbsolutePath());
- } catch (IOException e) {
- throw new ItemStreamException(e);
- }
-
- }
-
- @Override
- public void update(ExecutionContext executionContext) throws ItemStreamException {
-
- executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex);
- }
+ ExecutionContext context = new ExecutionContext();
+ reader.open(context);
- @Override
- public void close() throws ItemStreamException {
+ String result = reader.read();
- try {
- this.ds.close();
- } catch (IOException e) {
- throw new ItemStreamException(e);
- }
+ assertNotNull(result);
}
}