You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by fg...@apache.org on 2018/01/23 16:44:25 UTC
[28/32] incubator-sdap-ningester git commit: getting ready for Apache
commit
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByDimensionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByDimensionTest.java b/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByDimensionTest.java
new file mode 100644
index 0000000..d6a2ac9
--- /dev/null
+++ b/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByDimensionTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sdap.ningester.datatiler;
+
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class SliceFileByDimensionTest {
+
+ @Test
+ public void testGenerateTileBoundrySlices() {
+
+ SliceFileByDimension slicer = new SliceFileByDimension();
+ slicer.setSliceByDimension("NUMROWS");
+
+ Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>();
+ dimensionNameToLength.put("NUMROWS", 3163);
+ dimensionNameToLength.put("NUMCELLS", 82);
+
+ List<String> result = slicer.generateTileBoundrySlices("NUMROWS", dimensionNameToLength);
+
+ assertThat(result.size(), is(3163));
+ assertThat(result, hasItems("NUMROWS:0:1,NUMCELLS:0:82", "NUMROWS:1:2,NUMCELLS:0:82", "NUMROWS:3162:3163,NUMCELLS:0:82"));
+
+ }
+
+ @Test
+ public void testGenerateTileBoundrySlices2() {
+
+ SliceFileByDimension slicer = new SliceFileByDimension();
+ slicer.setSliceByDimension("NUMROWS");
+
+ Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>();
+ dimensionNameToLength.put("NUMROWS", 2);
+ dimensionNameToLength.put("NUMCELLS", 82);
+
+ List<String> result = slicer.generateTileBoundrySlices("NUMROWS", dimensionNameToLength);
+
+ assertThat(result.size(), is(2));
+ assertThat(result, hasItems("NUMROWS:0:1,NUMCELLS:0:82", "NUMROWS:1:2,NUMCELLS:0:82"));
+
+ }
+
+ @Test
+ public void testGenerateSlicesByInteger() throws IOException {
+
+
+ Integer expectedTiles = 1624;
+
+ SliceFileByDimension slicer = new SliceFileByDimension();
+ slicer.setDimensions(Arrays.asList("0", "1"));
+ slicer.setSliceByDimension("1");
+
+ Resource testResource = new ClassPathResource("granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5");
+
+ List<String> results = slicer.generateSlices(testResource.getFile());
+
+ assertThat(results.size(), is(expectedTiles));
+
+ }
+
+ @Test
+ public void testGenerateSlicesByName() throws IOException {
+
+
+ Integer expectedTiles = 3163;
+
+ SliceFileByDimension slicer = new SliceFileByDimension();
+ slicer.setDimensions(Arrays.asList("NUMROWS", "NUMCELLS"));
+ slicer.setSliceByDimension("NUMROWS");
+
+ Resource testResource = new ClassPathResource("granules/ascat_20121029_010301_metopb_00588_eps_o_coa_2101_ovw.l2.nc");
+
+ List<String> results = slicer.generateSlices(testResource.getFile());
+
+ assertThat(results.size(), is(expectedTiles));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesiredTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesiredTest.java b/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesiredTest.java
new file mode 100644
index 0000000..60cc18b
--- /dev/null
+++ b/src/test/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesiredTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.datatiler;
+
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+
+public class SliceFileByTilesDesiredTest {
+
+ @Test
+ public void testGenerateChunkBoundrySlicesWithDivisibileTiles() {
+
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+
+ Integer tilesDesired = 4;
+
+ Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>();
+ dimensionNameToLength.put("lat", 8);
+ dimensionNameToLength.put("lon", 8);
+
+ List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength);
+
+ assertEquals(tilesDesired.intValue(), result.size());
+
+ String[] expected = new String[]{
+ "lat:0:4,lon:0:4",
+ "lat:0:4,lon:4:8",
+ "lat:4:8,lon:0:4",
+ "lat:4:8,lon:4:8"};
+ assertThat(result, containsInAnyOrder(expected));
+ assertThat(result, contains(expected));
+
+ }
+
+ @Test
+ public void testGenerateChunkBoundrySlicesWithNonDivisibileTiles() {
+
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+
+ Integer tilesDesired = 5;
+
+ Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>();
+ dimensionNameToLength.put("lat", 8);
+ dimensionNameToLength.put("lon", 8);
+
+ List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength);
+
+ assertEquals(9, result.size());
+
+ String[] expected = new String[]{
+ "lat:0:3,lon:0:3",
+ "lat:0:3,lon:3:6",
+ "lat:0:3,lon:6:8",
+ "lat:3:6,lon:0:3",
+ "lat:3:6,lon:3:6",
+ "lat:3:6,lon:6:8",
+ "lat:6:8,lon:0:3",
+ "lat:6:8,lon:3:6",
+ "lat:6:8,lon:6:8"};
+ assertThat(result, containsInAnyOrder(expected));
+ assertThat(result, contains(expected));
+
+ }
+
+ @Test
+ public void testAddTimeDimension() {
+
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+ slicer.setTimeDimension("time");
+
+ Integer tilesDesired = 4;
+ Integer timeLen = 3;
+
+ Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>();
+ dimensionNameToLength.put("lat", 8);
+ dimensionNameToLength.put("lon", 8);
+
+ List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength);
+ result = slicer.addTimeDimension(result, timeLen);
+
+ assertEquals(tilesDesired * timeLen, result.size());
+
+ String[] expected = new String[]{
+ "time:0:1,lat:0:4,lon:0:4",
+ "time:1:2,lat:0:4,lon:0:4",
+ "time:2:3,lat:0:4,lon:0:4",
+
+ "time:0:1,lat:0:4,lon:4:8",
+ "time:1:2,lat:0:4,lon:4:8",
+ "time:2:3,lat:0:4,lon:4:8",
+
+ "time:0:1,lat:4:8,lon:0:4",
+ "time:1:2,lat:4:8,lon:0:4",
+ "time:2:3,lat:4:8,lon:0:4",
+
+ "time:0:1,lat:4:8,lon:4:8",
+ "time:1:2,lat:4:8,lon:4:8",
+ "time:2:3,lat:4:8,lon:4:8"};
+ assertThat(result, containsInAnyOrder(expected));
+ assertThat(result, contains(expected));
+
+ }
+
+ @Test
+ public void testGenerateChunkBoundrySlicesWithMurDimensions() {
+
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+
+ Integer tilesDesired = 5184;
+
+ Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>();
+ dimensionNameToLength.put("lat", 17999);
+ dimensionNameToLength.put("lon", 36000);
+
+ List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength);
+
+ assertEquals(tilesDesired + 72, result.size());
+
+ assertThat(result, hasItems("lat:0:249,lon:0:500", "lat:0:249,lon:500:1000", "lat:17928:17999,lon:35500:36000"));
+
+ }
+
+ @Test
+ public void testAddTimeDimensionWithMurDimensionsAndTime() {
+
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+
+ Integer tilesDesired = 5184;
+
+ Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>();
+ dimensionNameToLength.put("lat", 17999);
+ dimensionNameToLength.put("lon", 36000);
+
+ slicer.setTimeDimension("time");
+
+ List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength);
+ result = slicer.addTimeDimension(result, 1);
+
+ assertEquals(tilesDesired + 72, result.size());
+
+ assertThat(result, hasItems("time:0:1,lat:0:249,lon:0:500", "time:0:1,lat:0:249,lon:500:1000", "time:0:1,lat:17928:17999,lon:35500:36000"));
+
+ }
+
+ @Test
+ public void testGenerateSlicesCcmp() throws IOException {
+ Integer tilesDesired = 270;
+ Integer expectedTiles = 289 * 4; // 4 time slices and 289 tiles per time slice
+
+ SliceFileByTilesDesired slicer = new SliceFileByTilesDesired();
+ slicer.setTilesDesired(tilesDesired);
+ slicer.setDimensions(Arrays.asList("latitude", "longitude"));
+ slicer.setTimeDimension("time");
+
+ Resource testResource = new ClassPathResource("granules/CCMP_Wind_Analysis_20050101_V02.0_L3.0_RSS.nc");
+
+ List<String> results = slicer.generateSlices(testResource.getFile());
+
+ assertThat(results.size(), is(expectedTiles));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/processors/AddDatasetNameTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sdap/ningester/processors/AddDatasetNameTest.java b/src/test/java/org/apache/sdap/ningester/processors/AddDatasetNameTest.java
new file mode 100644
index 0000000..293d382
--- /dev/null
+++ b/src/test/java/org/apache/sdap/ningester/processors/AddDatasetNameTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.processors;
+
+import org.apache.sdap.nexusproto.NexusTile;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class AddDatasetNameTest {
+
+ @Test
+ public void testDatasetName() {
+
+ String datasetName = "testDataset";
+
+ NexusTile input = NexusTile.newBuilder().build();
+
+ AddDatasetName processor = new AddDatasetName(datasetName);
+
+ NexusTile result = processor.addDatasetName(input);
+
+ assertThat(result.getSummary().getDatasetName(), is(datasetName));
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java b/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
new file mode 100644
index 0000000..d43e0ec
--- /dev/null
+++ b/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.processors;
+
+import org.apache.sdap.nexusproto.NexusTile;
+import org.apache.sdap.nexusproto.TileSummary;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.beans.HasPropertyWithValue.hasProperty;
+
+public class AddDayOfYearAttributeTest {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testSuccessfulMatch() {
+ String regex = "^(\\d{3})";
+ String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc";
+ NexusTile nexusTile = NexusTile.newBuilder().setSummary(
+ TileSummary.newBuilder()
+ .setGranule(granuleName)
+ .build()
+ ).build();
+
+ AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex);
+
+ NexusTile result = processor.setDayOfYearFromGranuleName(nexusTile);
+
+ assertThat(result.getSummary().getGlobalAttributesList(), contains(
+ hasProperty("name", is("day_of_year_i"))
+ ));
+
+ String actualDayOfYear = result.getSummary().getGlobalAttributes(0).getValues(0);
+ assertThat(actualDayOfYear, is("001"));
+ }
+
+ @Test()
+ public void testUnsuccessfulMatch() {
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("regex did not match granuleName.");
+
+ String regex = "^(\\d{3})";
+ String granuleName = "L4_5day_avhrr_clim_sst_pixelMean.nc";
+ NexusTile nexusTile = NexusTile.newBuilder().setSummary(
+ TileSummary.newBuilder()
+ .setGranule(granuleName)
+ .build()
+ ).build();
+
+ AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex);
+
+ processor.setDayOfYearFromGranuleName(nexusTile);
+ }
+
+ @Test()
+ public void testUnsuccessfulGroupMatch() {
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("regex does not have exactly one capturing group.");
+
+ String regex = "^\\d{3}";
+ String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc";
+ NexusTile nexusTile = NexusTile.newBuilder().setSummary(
+ TileSummary.newBuilder()
+ .setGranule(granuleName)
+ .build()
+ ).build();
+
+ AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex);
+
+ processor.setDayOfYearFromGranuleName(nexusTile);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java b/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
new file mode 100644
index 0000000..4842290
--- /dev/null
+++ b/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sdap.ningester.processors;
+
+import org.apache.sdap.nexusproto.GridTile;
+import org.apache.sdap.nexusproto.NexusTile;
+import org.apache.sdap.nexusproto.TileData;
+import org.apache.sdap.nexusproto.TileSummary;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.text.ParseException;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.isA;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+
+public class AddTimeFromGranuleNameTest {
+
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testSuccessfulMatch() {
+ String regex = "^.*(\\d{7})\\.";
+ String dateFormat = "yyyyDDD";
+
+ String granuleName = "A2012001.L3m_DAY_NSST_sst_4km.nc";
+ Long expectedTime = 1325376000L; // 01/01/2012 00:00:00 in epoch time
+ NexusTile nexusTile = NexusTile.newBuilder().setSummary(
+ TileSummary.newBuilder()
+ .setGranule(granuleName)
+ .build()
+ ).setTile(
+ TileData.newBuilder()
+ .setGridTile(
+ GridTile.newBuilder(
+
+ ).build()
+ ).build()
+ ).build();
+
+ AddTimeFromGranuleName processor = new AddTimeFromGranuleName(regex, dateFormat);
+
+ NexusTile result = processor.setTimeFromGranuleName(nexusTile);
+
+ assertThat(result.getTile().getGridTile().getTime(), is(expectedTime));
+ assertThat(result.getSummary().getStats().getMinTime(), is(expectedTime));
+ assertThat(result.getSummary().getStats().getMaxTime(), is(expectedTime));
+ }
+
+ @Test
+ public void testUnparseable() {
+ String regex = "^.*(\\d{7})\\.";
+ String dateFormat = "yyyyDDDss";
+
+ String granuleName = "A2012001.L3m_DAY_NSST_sst_4km.nc";
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectCause(isA(ParseException.class));
+
+ NexusTile nexusTile = NexusTile.newBuilder().setSummary(
+ TileSummary.newBuilder()
+ .setGranule(granuleName)
+ .build()
+ ).setTile(
+ TileData.newBuilder()
+ .setGridTile(
+ GridTile.newBuilder(
+
+ ).build()
+ ).build()
+ ).build();
+
+ AddTimeFromGranuleName processor = new AddTimeFromGranuleName(regex, dateFormat);
+
+ NexusTile result = processor.setTimeFromGranuleName(nexusTile);
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/processors/GenerateTileIdTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sdap/ningester/processors/GenerateTileIdTest.java b/src/test/java/org/apache/sdap/ningester/processors/GenerateTileIdTest.java
new file mode 100644
index 0000000..bd9deff
--- /dev/null
+++ b/src/test/java/org/apache/sdap/ningester/processors/GenerateTileIdTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.processors;
+
+import org.apache.sdap.nexusproto.NexusTile;
+import org.apache.sdap.nexusproto.TileData;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class GenerateTileIdTest {
+
+
+ @Test
+ public void testGenerateId() {
+
+ String granuleFileName = "file:/path/to/some/file/19960421120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc";
+ String sectionSpec = "time:0:1,lat:140:160,lon:640:680";
+
+ String expectedId = "c031a9c4-9e1d-32e9-9d5c-d2497ce74920";
+
+ NexusTile.Builder inputTileBuilder = NexusTile.newBuilder();
+ inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName);
+ inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec);
+ inputTileBuilder.setTile(TileData.newBuilder());
+
+ GenerateTileId processor = new GenerateTileId();
+
+ NexusTile result = processor.addTileId(inputTileBuilder.build());
+
+ assertThat(result.getSummary().getTileId(), is(expectedId));
+ assertThat(result.getTile().getTileId(), is(expectedId));
+
+ }
+
+ @Test
+ public void testGenerateIdWithSalt() {
+
+ String granuleFileName = "file:/path/to/some/file/CCMP_Wind_Analysis_19990928_V02.0_L3.0_RSS.nc";
+ String sectionSpec = "time:3:4,longitude:174:261,latitude:152:190";
+ String salt = "wind_u";
+
+ String expectedId = "48da50ef-e92c-3562-89f9-470561a06482";
+
+ NexusTile.Builder inputTileBuilder = NexusTile.newBuilder();
+ inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName);
+ inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec);
+ inputTileBuilder.setTile(TileData.newBuilder());
+
+ GenerateTileId processor = new GenerateTileId();
+ processor.setSalt(salt);
+
+ NexusTile result = processor.addTileId(inputTileBuilder.build());
+
+ assertThat(result.getSummary().getTileId(), is(expectedId));
+ assertThat(result.getTile().getTileId(), is(expectedId));
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/writer/SolrStoreTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sdap/ningester/writer/SolrStoreTest.java b/src/test/java/org/apache/sdap/ningester/writer/SolrStoreTest.java
new file mode 100644
index 0000000..cbe85ac
--- /dev/null
+++ b/src/test/java/org/apache/sdap/ningester/writer/SolrStoreTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.writer;
+
+import org.apache.sdap.nexusproto.TileSummary;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class SolrStoreTest {
+
+ @Test
+ public void testGetSolrDocFromTileSummary() {
+ SolrStore solrStore = new SolrStore(null);
+
+ TileSummary tileSummary = TileSummary.newBuilder()
+ .setTileId("1")
+ .setBbox(TileSummary.BBox.newBuilder()
+ .setLatMin(51)
+ .setLatMax(55)
+ .setLonMin(22)
+ .setLonMax(30)
+ .build())
+ .setDatasetName("test")
+ .setDatasetUuid("4")
+ .setDataVarName("sst")
+ .setGranule("test.nc")
+ .setSectionSpec("0:1,0:1")
+ .setStats(TileSummary.DataStats.newBuilder()
+ .setCount(10)
+ .setMax(50)
+ .setMin(50)
+ .setMean(50)
+ .setMaxTime(1429142399)
+ .setMinTime(1429142399)
+ .build())
+ .build();
+
+ SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary);
+
+ assertEquals("2015-04-15T23:59:59Z", doc.get("tile_min_time_dt").getValue());
+ assertEquals("2015-04-15T23:59:59Z", doc.get("tile_max_time_dt").getValue());
+ assertEquals("sea_surface_temp", doc.get("table_s").getValue());
+ assertEquals("POLYGON((22.000 51.000, 30.000 51.000, 30.000 55.000, 22.000 55.000, 22.000 51.000))", doc.get("geo").getValue());
+ assertEquals("1", doc.get("id").getValue());
+ assertEquals("4", doc.get("dataset_id_s").getValue());
+ assertEquals("0:1,0:1", doc.get("sectionSpec_s").getValue());
+ assertEquals("test", doc.get("dataset_s").getValue());
+ assertEquals("test.nc", doc.get("granule_s").getValue());
+ assertEquals("sst", doc.get("tile_var_name_s").getValue());
+ assertEquals(22.0f, (Float) doc.get("tile_min_lon").getValue(), 0.01f);
+ assertEquals(30.0f, (Float) doc.get("tile_max_lon").getValue(), 0.01f);
+ assertEquals(51.0f, (Float) doc.get("tile_min_lat").getValue(), 0.01f);
+ assertEquals(55.0f, (Float) doc.get("tile_max_lat").getValue(), 0.01f);
+ assertEquals(50.0f, (Float) doc.get("tile_min_val_d").getValue(), 0.01f);
+ assertEquals(50.0f, (Float) doc.get("tile_max_val_d").getValue(), 0.01f);
+ assertEquals(50.0f, (Float) doc.get("tile_avg_val_d").getValue(), 0.01f);
+ assertEquals(10, doc.get("tile_count_i").getValue());
+ assertEquals("test!1", doc.get("solr_id_s").getValue());
+ }
+
+ @Test
+ public void testGeoIsPointWhenLatMinMaxEqualAndLonMinMaxEqual() {
+ SolrStore solrStore = new SolrStore(null);
+
+ TileSummary tileSummary = TileSummary.newBuilder()
+ .setBbox(TileSummary.BBox.newBuilder()
+ .setLatMin(51)
+ .setLatMax(51)
+ .setLonMin(22)
+ .setLonMax(22)
+ .build())
+ .build();
+
+ SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary);
+
+ assertEquals("POINT(22.000 51.000)", doc.get("geo").getValue());
+ }
+
+ @Test
+ public void testGeoIsLineStringWhenLatMinMaxEqualAndLonMinMaxNotEqual() {
+ SolrStore solrStore = new SolrStore(null);
+
+ TileSummary tileSummary = TileSummary.newBuilder()
+ .setBbox(TileSummary.BBox.newBuilder()
+ .setLatMin(51)
+ .setLatMax(51)
+ .setLonMin(22)
+ .setLonMax(29)
+ .build())
+ .build();
+
+ SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary);
+
+ assertEquals("LINESTRING (22.000 51.000, 29.000 51.000)", doc.get("geo").getValue());
+ }
+
+ @Test
+ public void testGeoIsLineStringWhenLatMinMaxNotEqualAndLonMinMaxEqual() {
+ SolrStore solrStore = new SolrStore(null);
+
+ TileSummary tileSummary = TileSummary.newBuilder()
+ .setBbox(TileSummary.BBox.newBuilder()
+ .setLatMin(51)
+ .setLatMax(59)
+ .setLonMin(22)
+ .setLonMax(22)
+ .build())
+ .build();
+
+ SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary);
+
+ assertEquals("LINESTRING (22.000 51.000, 22.000 59.000)", doc.get("geo").getValue());
+ }
+
+ @Test
+ public void testGeoIsLineStringWhenLatMinMaxAlmostEqualAndLonMinMaxNotEqual() {
+ SolrStore solrStore = new SolrStore(null);
+
+ TileSummary tileSummary = TileSummary.newBuilder()
+ .setBbox(TileSummary.BBox.newBuilder()
+ .setLatMin(-56.135883f)
+ .setLatMax(-56.135674f)
+ .setLonMin(-9.229431f)
+ .setLonMax(-8.934967f)
+ .build())
+ .build();
+
+ SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary);
+
+ assertEquals("LINESTRING (-9.229 -56.136, -8.935 -56.136)", doc.get("geo").getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java
----------------------------------------------------------------------
diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java
deleted file mode 100644
index 2822629..0000000
--- a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- *****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.testjobs;
-
-import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties;
-import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.batch.core.JobExecution;
-import org.springframework.batch.core.JobParameters;
-import org.springframework.batch.core.JobParametersBuilder;
-import org.springframework.batch.core.StepExecution;
-import org.springframework.batch.test.JobLauncherTestUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.context.TestConfiguration;
-import org.springframework.context.annotation.Bean;
-import org.springframework.data.cassandra.core.CassandraTemplate;
-import org.springframework.data.solr.core.SolrTemplate;
-import org.springframework.data.solr.core.query.SimpleQuery;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.TestPropertySource;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import static gov.nasa.jpl.nexus.ningester.testjobs.TestUtils.assertEqualsEventually;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-@RunWith(SpringRunner.class)
-@SpringBootTest
-@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/AvhrrJobTest.yml"})
-@ActiveProfiles({"test", "cassandra", "solr"})
-public class AvhrrJobTest {
-
- @Autowired
- JobLauncherTestUtils jobLauncherTestUtils;
- @Autowired
- CassandraTemplate cassandraTemplate;
- @Autowired
- SolrTemplate solrTemplate;
- @Autowired
- DatasourceProperties datasourceProperties;
- @Autowired
- ApplicationProperties applicationProperties;
-
- @Before
- public void emptyDatabase() {
- solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("*:*"));
- cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName());
- }
-
- @Test
- public void testJobCompletes() throws Exception {
-
- JobParameters jobParameters = new JobParametersBuilder()
- .addString("granule", "classpath:granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc")
- .toJobParameters();
-
- JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters);
-
- assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED"));
- StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next();
- assertThat(stepExecution.getReadCount(), is(5184));
- assertThat(stepExecution.getWriteCount(), is(3904));
- assertThat(stepExecution.getFilterCount(), is(1280));
-
- assertEqualsEventually(3904L,
- () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(),
- new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())),
- 3);
-
- long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName());
-
- assertThat(cassandraCount, is(3904L));
- }
-
- @TestConfiguration
- static class NingesterApplicationTestsConfig {
-
- @Bean
- JobLauncherTestUtils jobLauncherTestUtils() {
- return new JobLauncherTestUtils();
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java
----------------------------------------------------------------------
diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java
deleted file mode 100644
index 89d01eb..0000000
--- a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*****************************************************************************
- * Copyright (c) 2017 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.testjobs;
-
-import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties;
-import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.batch.core.JobExecution;
-import org.springframework.batch.core.JobParameters;
-import org.springframework.batch.core.JobParametersBuilder;
-import org.springframework.batch.core.StepExecution;
-import org.springframework.batch.test.JobLauncherTestUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.context.TestConfiguration;
-import org.springframework.context.annotation.Bean;
-import org.springframework.data.cassandra.core.CassandraTemplate;
-import org.springframework.data.solr.core.SolrTemplate;
-import org.springframework.data.solr.core.query.SimpleQuery;
-import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.TestPropertySource;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import static gov.nasa.jpl.nexus.ningester.testjobs.TestUtils.assertEqualsEventually;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-@RunWith(SpringRunner.class)
-@SpringBootTest
-@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/SmapJobTest.yml"})
-@ActiveProfiles({"test", "cassandra", "solr"})
-public class SmapJobTest {
-
- @Autowired
- JobLauncherTestUtils jobLauncherTestUtils;
- @Autowired
- CassandraTemplate cassandraTemplate;
- @Autowired
- SolrTemplate solrTemplate;
- @Autowired
- DatasourceProperties datasourceProperties;
- @Autowired
- ApplicationProperties applicationProperties;
-
- @Before
- public void emptyDatabase() {
- solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("*:*"));
- cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName());
- }
-
- @Test
- public void testJobCompletes() throws Exception {
-
- JobParameters jobParameters = new JobParametersBuilder()
- .addString("granule", "classpath:granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5")
- .toJobParameters();
-
- JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters);
-
- assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED"));
- StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next();
- assertThat(stepExecution.getReadCount(), is(1624));
- assertThat(stepExecution.getWriteCount(), is(990));
- assertThat(stepExecution.getFilterCount(), is(634));
-
- assertEqualsEventually(990L,
- () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(),
- new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())),
- 2);
-
- long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName());
-
- assertThat(cassandraCount, is(990L));
-
-
- }
-
- @TestConfiguration
- static class NingesterApplicationTestsConfig {
-
- @Bean
- JobLauncherTestUtils jobLauncherTestUtils() {
- return new JobLauncherTestUtils();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java
deleted file mode 100644
index 6ce6850..0000000
--- a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- ****************************************************************************
- * Copyright (c) 2018 Jet Propulsion Laboratory,
- * California Institute of Technology. All rights reserved
- *****************************************************************************/
-
-package gov.nasa.jpl.nexus.ningester.testjobs;
-
-import java.util.function.Supplier;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-
-public class TestUtils {
-
- public static <T> void assertEqualsEventually(T expected, Supplier<T> actualSupplier, int secondsToWait) throws InterruptedException {
- int n = 0;
- int iterations = 10 * secondsToWait;
- int sleepTimeMillis = 100;
-
- T suppliedValue = actualSupplier.get();
- while (!suppliedValue.equals(expected) && n++ < iterations) {
- Thread.sleep(sleepTimeMillis);
- suppliedValue = actualSupplier.get();
- }
- assertThat("Did not equal after waiting " + (iterations * sleepTimeMillis / 1000) + " seconds.", suppliedValue, is(expected));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java
----------------------------------------------------------------------
diff --git a/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java b/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java
new file mode 100644
index 0000000..498136f
--- /dev/null
+++ b/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.testjobs;
+
+import org.apache.sdap.ningester.configuration.properties.ApplicationProperties;
+import org.apache.sdap.ningester.configuration.properties.DatasourceProperties;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.test.JobLauncherTestUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.data.cassandra.core.CassandraTemplate;
+import org.springframework.data.solr.core.SolrTemplate;
+import org.springframework.data.solr.core.query.SimpleQuery;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.apache.sdap.ningester.testjobs.TestUtils.assertEqualsEventually;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/AvhrrJobTest.yml"})
+@ActiveProfiles({"test", "cassandra", "solr"})
+public class AvhrrJobTest {
+
+ @Autowired
+ JobLauncherTestUtils jobLauncherTestUtils;
+ @Autowired
+ CassandraTemplate cassandraTemplate;
+ @Autowired
+ SolrTemplate solrTemplate;
+ @Autowired
+ DatasourceProperties datasourceProperties;
+ @Autowired
+ ApplicationProperties applicationProperties;
+
+ @Before
+ public void emptyDatabase() {
+ solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("*:*"));
+ cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName());
+ }
+
+ @Test
+ public void testJobCompletes() throws Exception {
+
+ JobParameters jobParameters = new JobParametersBuilder()
+ .addString("granule", "classpath:granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc")
+ .toJobParameters();
+
+ JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters);
+
+ assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED"));
+ StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next();
+ assertThat(stepExecution.getReadCount(), is(5184));
+ assertThat(stepExecution.getWriteCount(), is(3904));
+ assertThat(stepExecution.getFilterCount(), is(1280));
+
+ assertEqualsEventually(3904L,
+ () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(),
+ new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())),
+ 3);
+
+ long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName());
+
+ assertThat(cassandraCount, is(3904L));
+ }
+
+ @TestConfiguration
+ static class NingesterApplicationTestsConfig {
+
+ @Bean
+ JobLauncherTestUtils jobLauncherTestUtils() {
+ return new JobLauncherTestUtils();
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/testJobs/java/org/apache/sdap/ningester/testjobs/SmapJobTest.java
----------------------------------------------------------------------
diff --git a/src/testJobs/java/org/apache/sdap/ningester/testjobs/SmapJobTest.java b/src/testJobs/java/org/apache/sdap/ningester/testjobs/SmapJobTest.java
new file mode 100644
index 0000000..aab2eae
--- /dev/null
+++ b/src/testJobs/java/org/apache/sdap/ningester/testjobs/SmapJobTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.testjobs;
+
+import org.apache.sdap.ningester.configuration.properties.ApplicationProperties;
+import org.apache.sdap.ningester.configuration.properties.DatasourceProperties;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.test.JobLauncherTestUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.data.cassandra.core.CassandraTemplate;
+import org.springframework.data.solr.core.SolrTemplate;
+import org.springframework.data.solr.core.query.SimpleQuery;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/SmapJobTest.yml"})
+@ActiveProfiles({"test", "cassandra", "solr"})
+public class SmapJobTest {
+
+ @Autowired
+ JobLauncherTestUtils jobLauncherTestUtils;
+ @Autowired
+ CassandraTemplate cassandraTemplate;
+ @Autowired
+ SolrTemplate solrTemplate;
+ @Autowired
+ DatasourceProperties datasourceProperties;
+ @Autowired
+ ApplicationProperties applicationProperties;
+
+ @Before
+ public void emptyDatabase() {
+ solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("*:*"));
+ cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName());
+ }
+
+ @Test
+ public void testJobCompletes() throws Exception {
+
+ JobParameters jobParameters = new JobParametersBuilder()
+ .addString("granule", "classpath:granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5")
+ .toJobParameters();
+
+ JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters);
+
+ assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED"));
+ StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next();
+ assertThat(stepExecution.getReadCount(), is(1624));
+ assertThat(stepExecution.getWriteCount(), is(990));
+ assertThat(stepExecution.getFilterCount(), is(634));
+
+ TestUtils.assertEqualsEventually(990L,
+ () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(),
+ new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())),
+ 2);
+
+ long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName());
+
+ assertThat(cassandraCount, is(990L));
+
+
+ }
+
+ @TestConfiguration
+ static class NingesterApplicationTestsConfig {
+
+ @Bean
+ JobLauncherTestUtils jobLauncherTestUtils() {
+ return new JobLauncherTestUtils();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/testJobs/java/org/apache/sdap/ningester/testjobs/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/testJobs/java/org/apache/sdap/ningester/testjobs/TestUtils.java b/src/testJobs/java/org/apache/sdap/ningester/testjobs/TestUtils.java
new file mode 100644
index 0000000..6bd4932
--- /dev/null
+++ b/src/testJobs/java/org/apache/sdap/ningester/testjobs/TestUtils.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.ningester.testjobs;
+
+import java.util.function.Supplier;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+public class TestUtils {
+
+ public static <T> void assertEqualsEventually(T expected, Supplier<T> actualSupplier, int secondsToWait) throws InterruptedException {
+ int n = 0;
+ int iterations = 10 * secondsToWait;
+ int sleepTimeMillis = 100;
+
+ T suppliedValue = actualSupplier.get();
+ while (!suppliedValue.equals(expected) && n++ < iterations) {
+ Thread.sleep(sleepTimeMillis);
+ suppliedValue = actualSupplier.get();
+ }
+ assertThat("Did not equal after waiting " + (iterations * sleepTimeMillis / 1000) + " seconds.", suppliedValue, is(expected));
+ }
+}