You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sis.apache.org by de...@apache.org on 2018/11/22 19:57:09 UTC
[sis] 02/02: Partial decoding of HYCOM netCDF files.
This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit 367df41cd5dfe7e156402ceb147442cb62f12e26
Author: Martin Desruisseaux <ma...@geomatys.com>
AuthorDate: Thu Nov 22 20:56:32 2018 +0100
Partial decoding of HYCOM netCDF files.
---
.../org/apache/sis/coverage/grid/GridGeometry.java | 2 +-
.../java/org/apache/sis/internal/netcdf/Axis.java | 48 ++++----
.../java/org/apache/sis/internal/netcdf/Grid.java | 2 +
.../org/apache/sis/internal/netcdf/Variable.java | 40 ++++++-
.../sis/internal/netcdf/impl/ChannelDecoder.java | 83 ++++++++-----
.../apache/sis/internal/netcdf/impl/GridInfo.java | 31 ++++-
.../org/apache/sis/internal/netcdf/impl/HYCOM.java | 129 +++++++++++++++++++++
.../sis/internal/netcdf/impl/VariableInfo.java | 38 +++++-
.../sis/internal/netcdf/ucar/VariableWrapper.java | 12 +-
.../sis/storage/netcdf/NetcdfStoreProvider.java | 4 +-
10 files changed, 325 insertions(+), 64 deletions(-)
diff --git a/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java b/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java
index 04b34cc..e19eb8f 100644
--- a/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java
+++ b/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java
@@ -980,7 +980,7 @@ public class GridGeometry implements Serializable {
while (nonLinearDimensions != 0) {
final int i = Long.numberOfTrailingZeros(nonLinearDimensions);
nonLinearDimensions &= ~(1L << i);
- buffer.append(separator).append(cs != null ? cs.getAxis(i).getName() : String.valueOf(i));
+ buffer.append(separator).append(cs != null ? cs.getAxis(i).getName().getCode() : String.valueOf(i));
separator = ", ";
}
buffer.append(lineSeparator);
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
index a1e533d..bae678d 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
@@ -356,28 +356,32 @@ public final class Axis extends NamedElement {
*/
if (sourceDimensions.length == 2) {
Vector data = coordinates.read();
- final int[] repetitions = data.repetitions(); // Detects repetitions as illustrated above.
- for (int i=0; i<sourceDimensions.length; i++) {
- final int srcDim = srcEnd - sourceDimensions[i]; // "Natural" order is reverse of netCDF order.
- final int length = sourceSizes[i];
- int step = 1;
- for (int j=0; j<sourceDimensions.length; j++) {
- int previous = srcEnd - sourceDimensions[j];
- if (previous < srcDim) step *= sourceSizes[j];
- }
- final boolean condition;
- switch (srcDim) {
- case 0: condition = repetitions.length > 1 && (repetitions[1] % length) == 0; break;
- case 1: condition = repetitions.length > 0 && (repetitions[0] % step) == 0; break;
- default: throw new AssertionError(); // I don't know yet how to generalize to n dimensions.
- }
- if (condition) { // Repetition length shall be grid size (or a multiple).
- data = data.subSampling(0, step, length);
- if (coordinates.trySetTransform(gridToCRS, srcDim, tgtDim, data)) {
- return true;
- } else {
- nonLinears.add(MathTransforms.interpolate(null, data.doubleValues()));
- return false;
+ if (!coordinates.readTriesToCompress() || data.getClass().getSimpleName().equals("RepeatedVector")) {
+ final int[] repetitions = data.repetitions(); // Detects repetitions as illustrated above.
+ if (repetitions.length != 0) {
+ for (int i=0; i<sourceDimensions.length; i++) {
+ final int srcDim = srcEnd - sourceDimensions[i]; // "Natural" order is reverse of netCDF order.
+ final int length = sourceSizes[i];
+ int step = 1;
+ for (int j=0; j<sourceDimensions.length; j++) {
+ int previous = srcEnd - sourceDimensions[j];
+ if (previous < srcDim) step *= sourceSizes[j];
+ }
+ final boolean condition;
+ switch (srcDim) {
+ case 0: condition = repetitions.length > 1 && (repetitions[1] % length) == 0; break;
+ case 1: condition = (repetitions[0] % step) == 0; break;
+ default: throw new AssertionError(); // I don't know yet how to generalize to n dimensions.
+ }
+ if (condition) { // Repetition length shall be grid size (or a multiple).
+ data = data.subSampling(0, step, length);
+ if (coordinates.trySetTransform(gridToCRS, srcDim, tgtDim, data)) {
+ return true;
+ } else {
+ nonLinears.add(MathTransforms.interpolate(null, data.doubleValues()));
+ return false;
+ }
+ }
}
}
}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
index e8a4c50..c8e1d2f 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
@@ -53,6 +53,8 @@ import org.apache.sis.util.NullArgumentException;
public abstract class Grid extends NamedElement {
/**
* The axes, created when first needed.
+ * The ordering of axes is based on the order in which dimensions are declared for variables using this grid.
+ * This is not necessarily the same order than the order in which variables are listed in the netCDF file.
*
* @see #getAxes()
*/
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
index 412d345..8d8ca0d 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
@@ -42,8 +42,11 @@ import org.apache.sis.util.resources.Errors;
public abstract class Variable extends NamedElement {
/**
* The pattern to use for parsing temporal units of the form "days since 1970-01-01 00:00:00".
+ *
+ * @see #parseUnit(String)
+ * @see Decoder#numberToDate(String, Number[])
*/
- protected static final Pattern TIME_PATTERN = Pattern.compile("(.+)\\Wsince\\W(.+)", Pattern.CASE_INSENSITIVE);
+ public static final Pattern TIME_UNIT_PATTERN = Pattern.compile("(.+)\\Wsince\\W(.+)", Pattern.CASE_INSENSITIVE);
/**
* Minimal number of dimension for accepting a variable as a coverage variable.
@@ -54,18 +57,23 @@ public abstract class Variable extends NamedElement {
* The unit of measurement, parsed from {@link #getUnitsString()} when first needed.
* We do not try to parse the unit at construction time because this variable may be
* never requested by the user.
+ *
+ * @see #getUnit()
*/
private Unit<?> unit;
/**
* If the unit is a temporal unit of the form "days since 1970-01-01 00:00:00", the epoch.
- * Otherwise {@code null}. This value can be set
+ * Otherwise {@code null}. This value can be set by subclasses as a side-effect of their
+ * {@link #parseUnit(String)} method implementation.
*/
protected Instant epoch;
/**
* Whether an attempt to parse the unit has already be done. This is used for avoiding
* to report the same failure many times when {@link #unit} stay null.
+ *
+ * @see #getUnit()
*/
private boolean unitParsed;
@@ -129,6 +137,26 @@ public abstract class Variable extends NamedElement {
protected abstract Unit<?> parseUnit(String symbols) throws Exception;
/**
+ * Sets the unit of measurement and the epoch to the same value than the given variable.
+ * This method is not used in CF-compliant files; it is reserved for the handling of some
+ * particular conventions, for example HYCOM.
+ *
+ * @param other the variable from which to copy unit and epoch, or {@code null} if none.
+ * @param overwrite if non-null, set to the given unit instead than the unit of {@code other}.
+ * @return the epoch (may be {@code null}).
+ */
+ public final Instant setUnit(final Variable other, Unit<?> overwrite) {
+ if (other != null) {
+ unit = other.getUnit(); // May compute the epoch as a side effect.
+ epoch = other.epoch;
+ }
+ if (overwrite != null) {
+ unit = overwrite;
+ }
+ return epoch;
+ }
+
+ /**
* Returns the unit of measurement for this variable, or {@code null} if unknown.
* This method parse the units from {@link #getUnitsString()} when first needed
* and sets {@link #epoch} as a side-effect if the unit is temporal.
@@ -291,6 +319,14 @@ public abstract class Variable extends NamedElement {
public abstract String getAttributeString(final String attributeName);
/**
+ * Whether {@link #read()} invoked {@link Vector#compress(double)} on the returned vector.
+ * This information is used for avoiding to do twice some potentially costly operations.
+ *
+ * @return whether {@link #read()} invokes {@link Vector#compress(double)}.
+ */
+ protected abstract boolean readTriesToCompress();
+
+ /**
* Reads all the data for this variable and returns them as an array of a Java primitive type.
* Multi-dimensional variables are flattened as a one-dimensional array (wrapped in a vector).
* Example:
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
index c946987..033fcd0 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
@@ -25,11 +25,10 @@ import java.util.LinkedHashSet;
import java.util.LinkedHashMap;
import java.util.IdentityHashMap;
import java.util.List;
-import java.util.Arrays;
import java.util.ArrayList;
import java.util.Date;
import java.util.Locale;
-import java.util.regex.Pattern;
+import java.util.regex.Matcher;
import java.time.DateTimeException;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -111,14 +110,6 @@ public final class ChannelDecoder extends Decoder {
*/
static final Locale NAME_LOCALE = Locale.US;
- /**
- * The pattern to use for separating the component of a time unit.
- * An example of time unit is <cite>"days since 1970-01-01T00:00:00Z"</cite>.
- *
- * @see #numberToDate(String, Number[])
- */
- private static final Pattern TIME_UNIT_PATTERN = Pattern.compile("\\s+since\\s+", Pattern.CASE_INSENSITIVE);
-
/*
* NOTE: the names of the static constants below this point match the names used in the Backus-Naur Form (BNF)
* definitions in the netCDF Classic and 64-bit Offset Format (1.0) specification (link in class javdoc),
@@ -626,6 +617,18 @@ public final class ChannelDecoder extends Decoder {
return variables;
}
+ /**
+ * Checks and potentially modifies the content of this dataset for conventions other than CF-conventions.
+ * This method should be invoked after construction for handling the particularities of some datasets
+ * (HYCOM, …).
+ *
+ * @throws IOException if an error occurred while reading the channel.
+ * @throws DataStoreContentException if an error occurred while interpreting the netCDF file content.
+ */
+ public final void applyOtherConventions() throws IOException, DataStoreContentException {
+ HYCOM.convert(this, variables);
+ }
+
// --------------------------------------------------------------------------------------------
@@ -791,10 +794,10 @@ public final class ChannelDecoder extends Decoder {
@Override
public Date[] numberToDate(final String symbol, final Number... values) {
final Date[] dates = new Date[values.length];
- final String[] parts = TIME_UNIT_PATTERN.split(symbol);
- if (parts.length == 2) try {
- final UnitConverter converter = Units.valueOf(parts[0]).getConverterToAny(Units.MILLISECOND);
- final long epoch = StandardDateFormat.toDate(StandardDateFormat.FORMAT.parse(parts[1])).getTime();
+ final Matcher parts = Variable.TIME_UNIT_PATTERN.matcher(symbol);
+ if (parts.matches()) try {
+ final UnitConverter converter = Units.valueOf(parts.group(1)).getConverterToAny(Units.MILLISECOND);
+ final long epoch = StandardDateFormat.toDate(StandardDateFormat.FORMAT.parse(parts.group(2))).getTime();
for (int i=0; i<values.length; i++) {
final Number value = values[i];
if (value != null) {
@@ -851,7 +854,7 @@ public final class ChannelDecoder extends Decoder {
* First, find all variables which are used as coordinate system axis. The keys in the map are
* the grid dimensions which are the domain of the variable (i.e. the sources of the conversion
* from grid coordinates to CRS coordinates). For each key there is usually only one value, but
- * we try to make this code robust to unusual netCDF files.
+ * more complicated netCDF files (e.g. using two-dimensional localisation grids) also exist.
*/
final Map<Dimension, List<VariableInfo>> dimToAxes = new IdentityHashMap<>();
for (final VariableInfo variable : variables) {
@@ -862,25 +865,34 @@ public final class ChannelDecoder extends Decoder {
}
}
/*
- * For each variables, get the list of all axes associated to their dimensions. The association
- * is given by the above 'dimToVars' map. More than one variable may have the same dimensions,
- * and consequently the same axes, so we will remember the previously created instances in order
- * to share them.
+ * For each variable, get its list of axes. More than one variable may have the same list of axes,
+ * so we remember the previously created instances in order to share the grid geometry instances.
*/
final Set<VariableInfo> axes = new LinkedHashSet<>(4);
- final Map<List<Dimension>, GridInfo> dimsToGG = new LinkedHashMap<>();
+ final Map<GridInfo,GridInfo> shared = new LinkedHashMap<>();
nextVar: for (final VariableInfo variable : variables) {
if (variable.isCoordinateSystemAxis() || variable.dimensions.length == 0) {
continue;
}
- final List<Dimension> dimensions = Arrays.asList(variable.dimensions);
- GridInfo gridGeometry = dimsToGG.get(dimensions);
- if (gridGeometry == null) {
- /*
- * Found a new list of dimensions for which no axes have been created yet.
- * If and only if we can find all axes, then create the GridGeometryInfo.
- * This is a "all or nothing" operation.
- */
+ /*
+ * The axes can be inferred in two ways: if the variable contains a "coordinates" attribute,
+ * that attribute lists explicitly the variables to use as axes. Otherwise we have to infer
+ * the axes from the variable dimensions, using the 'dimToVars' map computed at the beginning
+ * of this method. If and only if we can find all axes, we create the GridGeometryInfo.
+ * This is a "all or nothing" operation.
+ */
+ final CharSequence[] coordinates = variable.getCoordinateVariables();
+ if (coordinates.length != 0) {
+ for (int i=coordinates.length; --i >= 0;) {
+ final VariableInfo axis = findVariable(coordinates[i].toString());
+ if (axis == null) {
+ axes.clear();
+ break;
+ }
+ axes.add(axis);
+ }
+ }
+ if (axes.isEmpty()) {
for (final Dimension dimension : variable.dimensions) {
final List<VariableInfo> axis = dimToAxes.get(dimension); // Should have only 1 element.
if (axis == null) {
@@ -889,13 +901,20 @@ nextVar: for (final VariableInfo variable : variables) {
}
axes.addAll(axis);
}
- gridGeometry = new GridInfo(variable.dimensions, axes.toArray(new VariableInfo[axes.size()]));
- dimsToGG.put(dimensions, gridGeometry);
- axes.clear();
+ }
+ /*
+ * Creates the grid geometry using the given domain and range,
+ * reusing existing instance if one exists.
+ */
+ GridInfo gridGeometry = new GridInfo(variable.dimensions, axes.toArray(new VariableInfo[axes.size()]));
+ GridInfo existing = shared.putIfAbsent(gridGeometry, gridGeometry);
+ if (existing != null) {
+ gridGeometry = existing;
}
variable.gridGeometry = gridGeometry;
+ axes.clear();
}
- gridGeometries = dimsToGG.values().toArray(new Grid[dimsToGG.size()]);
+ gridGeometries = shared.values().toArray(new Grid[shared.size()]);
}
return gridGeometries;
}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java
index 452a19a..033c74a 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java
@@ -17,6 +17,7 @@
package org.apache.sis.internal.netcdf.impl;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Locale;
import java.util.Map;
import java.util.HashMap;
@@ -29,6 +30,7 @@ import org.apache.sis.internal.netcdf.Grid;
import org.apache.sis.internal.netcdf.Resources;
import org.apache.sis.storage.DataStoreContentException;
import org.apache.sis.storage.DataStoreException;
+import org.apache.sis.measure.Units;
import ucar.nc2.constants.CF;
@@ -188,7 +190,7 @@ final class GridInfo extends Grid {
/*
* Process the variables in the order the appear in the sequence of bytes that make the netCDF files.
* This is often the same order than the indices, but not necessarily. The intent is to reduce the
- * amount of disk seek operations.
+ * amount of disk seek operations. Data loading may happen in this method through Axis constructor.
*/
final SortedMap<VariableInfo,Integer> variables = new TreeMap<>();
for (int i=0; i<range.length; i++) {
@@ -208,6 +210,11 @@ final class GridInfo extends Grid {
char abbreviation = getAxisType(axis.getAxisType());
if (abbreviation == 0) {
abbreviation = getAxisType(axis.getName());
+ if (abbreviation == 0) {
+ if (Units.isTemporal(axis.getUnit())) {
+ abbreviation = 't';
+ }
+ }
}
/*
* Get the grid dimensions (part of the "domain" in UCAR terminology) used for computing
@@ -245,4 +252,26 @@ final class GridInfo extends Grid {
final int n = v.dimensions[0].length;
return v.read().doubleValue(j + n*i);
}
+
+ /**
+ * Returns a hash code for this grid. A map of {@code GridInfo} is used by
+ * {@link ChannelDecoder#getGridGeometries()} for sharing existing instances.
+ */
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(domain) ^ Arrays.hashCode(range);
+ }
+
+ /**
+ * Compares the grid with the given object for equality.
+ */
+ @Override
+ public boolean equals(final Object other) {
+ if (other instanceof GridInfo) {
+ final GridInfo that = (GridInfo) other;
+ return Arrays.equals(domain, that.domain) &&
+ Arrays.equals(range, that.range);
+ }
+ return false;
+ }
}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/HYCOM.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/HYCOM.java
new file mode 100644
index 0000000..3e8ec40
--- /dev/null
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/HYCOM.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.internal.netcdf.impl;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+import org.apache.sis.math.Vector;
+import org.apache.sis.measure.Units;
+import org.apache.sis.internal.util.StandardDateFormat;
+import org.apache.sis.storage.DataStoreContentException;
+
+
+/**
+ * Handles particularity of HYCOM format. It is not yet clear whether those particularities are used elsewhere or not.
+ * We handle them in a separated class for now and may refactor later in a more general mechanism for providing extensions.
+ *
+ * @author Martin Desruisseaux (Geomatys)
+ * @version 1.0
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/SIS-315">SIS-315</a>
+ *
+ * @since 1.0
+ * @module
+ */
+final class HYCOM {
+ /**
+ * The pattern to use for identifying temporal units of the form "day as %Y%m%d.%f".
+ * "%Y" is year formatted as at least four digits, "%m" is month formatted as two digits,
+ * and "%d" is day of month formatted as two digits.
+ *
+ * Example: 20181017.0000 for 2018-10-17.
+ */
+ private static final Pattern DATE_PATTERN = Pattern.compile("days?\\s+as\\s+(?-i)%Y%m%d.*", Pattern.CASE_INSENSITIVE);
+
+ /**
+ * Do not allow instantiation of this class.
+ */
+ private HYCOM() {
+ }
+
+ /**
+ * If any variable uses the "day as %Y%m%d.%f" pseudo-units, converts to a number of days since the epoch.
+ * The epoch is taken from the unit of the dimension. Example of netCDF file header:
+ *
+ * {@preformat text
+ * dimensions:
+ * MT = UNLIMITED ; // (1 currently)
+ * Y = 3298 ;
+ * X = 4500 ;
+ * variables:
+ * double MT(MT) ;
+ * MT:long_name = "time" ;
+ * MT:units = "days since 1900-12-31 00:00:00" ;
+ * MT:calendar = "standard" ;
+ * MT:axis = "T" ;
+ * double Date(MT) ;
+ * Date:long_name = "date" ;
+ * Date:units = "day as %Y%m%d.%f" ;
+ * Date:C_format = "%13.4f" ;
+ * Date:FORTRAN_format = "(f13.4)" ;
+ * data:
+ * MT = 43024 ;
+ * Date = 20181017.0000 ;
+ * }
+ *
+ * In this example, the real units of {@code Date(MT)} will be taken from {@code MT(MT)}, which is
+ * "days since 1900-12-31 00:00:00".
+ */
+ static void convert(final ChannelDecoder decoder, final VariableInfo[] variables) throws IOException, DataStoreContentException {
+ Matcher matcher = null;
+ for (final VariableInfo variable : variables) {
+ if (variable.dimensions.length == 1) {
+ final String units = variable.getUnitsString();
+ if (units != null) {
+ if (matcher == null) {
+ matcher = DATE_PATTERN.matcher(units);
+ } else {
+ matcher.reset(units);
+ }
+ if (matcher.matches()) {
+ Instant epoch = variable.setUnit(decoder.findVariable(variable.dimensions[0].name), Units.DAY);
+ if (epoch == null) {
+ epoch = Instant.EPOCH;
+ }
+ final long origin = epoch.toEpochMilli();
+ /*
+ * Convert all dates into numbers of days since the epoch.
+ */
+ Vector values = variable.read();
+ final double[] times = new double[values.size()];
+ final GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone(StandardDateFormat.UTC), Locale.US);
+ calendar.clear();
+ for (int i=0; i<times.length; i++) {
+ double time = values.doubleValue(i); // Date encoded as a double (e.g. 20181017)
+ long date = (long) time; // Round toward zero.
+ time -= date; // Fractional part of the day.
+ int day = (int) (date % 100); date /= 100;
+ int month = (int) (date % 100); date /= 100;
+ calendar.set(Math.toIntExact(date), month - 1, day, 0, 0, 0);
+ date = calendar.getTimeInMillis() - origin; // Milliseconds since epoch.
+ time += date / (double) StandardDateFormat.MILLISECONDS_PER_DAY;
+ times[i] = time;
+ }
+ variable.setValues(times);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
index b2d6561..38054cc 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
@@ -300,7 +300,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
boolean isUnknown = false; // True if 'total' is actually unknown.
for (final VariableInfo variable : variables) {
// Opportunistically store names of all axes listed in "coordinates" attributes of all variables.
- referencedAsAxis.addAll(Arrays.asList(CharSequences.split(variable.getAttributeString(CF.COORDINATES), ' ')));
+ referencedAsAxis.addAll(Arrays.asList(variable.getCoordinateVariables()));
if (variable.isUnlimited()) {
final long paddedSize = variable.paddedSize();
unlimited[count++] = variable;
@@ -393,7 +393,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
*/
@Override
protected Unit<?> parseUnit(String symbols) {
- final Matcher parts = TIME_PATTERN.matcher(symbols);
+ final Matcher parts = TIME_UNIT_PATTERN.matcher(symbols);
if (parts.matches()) {
/*
* If we enter in this block, the unit is of the form "days since 1970-01-01 00:00:00".
@@ -452,6 +452,13 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
}
/**
+ * Returns the names of variables to use as axes for this variable, or an empty array if none.
+ */
+ final CharSequence[] getCoordinateVariables() {
+ return CharSequences.split(getAttributeString(CF.COORDINATES), ' ');
+ }
+
+ /**
* Returns the grid geometry for this variable, or {@code null} if this variable is not a data cube.
* The grid geometries are opportunistically cached in {@code VariableInfo} instances after they have
* been computed by {@link ChannelDecoder#getGridGeometries()}.
@@ -607,6 +614,26 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
}
/**
+ * Whether {@link #read()} invokes {@link Vector#compress(double)} on the returned vector.
+ *
+ * @return {@code true}.
+ */
+ @Override
+ protected boolean readTriesToCompress() {
+ return true;
+ }
+
+ /**
+ * Sets the values in this variable. The values are normally read from the netCDF file by the {@link #read()} method,
+ * but this {@code setValues(Object)} method may also be invoked if we want to overwrite those values.
+ *
+ * @param array the values as an array of primitive type (for example {@code float[]}.
+ */
+ final void setValues(final Object array) {
+ values = createDecimalVector(array, dataType.isUnsigned).compress(0);
+ }
+
+ /**
* Reads all the data for this variable and returns them as an array of a Java primitive type.
* Multi-dimensional variables are flattened as a one-dimensional array (wrapped in a vector).
* The vector is cached and returned as-is in all future invocation of this method.
@@ -630,7 +657,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
}
final Region region = new Region(upper, lower, upper, subsampling);
applyUnlimitedDimensionStride(region);
- values = createDecimalVector(reader.read(region), dataType.isUnsigned).compress(0);
+ setValues(reader.read(region));
}
return values;
}
@@ -664,10 +691,13 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
* @throws ArithmeticException if the size of the region to read exceeds {@link Integer#MAX_VALUE}, or other overflow occurs.
*/
@Override
- public Vector read(int[] areaLower, int[] areaUpper, int[] subsampling) throws IOException, DataStoreContentException {
+ public Vector read(int[] areaLower, int[] areaUpper, int[] subsampling) throws IOException, DataStoreException {
if (reader == null) {
throw new DataStoreContentException(unknownType());
}
+ if (values != null) {
+ throw new DataStoreException(); // TODO: create a view.
+ }
/*
* NetCDF sorts datas in reverse dimension order. Example:
*
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
index 6bc71dc..0f09b30 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
@@ -131,7 +131,7 @@ final class VariableWrapper extends Variable {
*/
@Override
protected Unit<?> parseUnit(final String symbols) throws Exception {
- if (TIME_PATTERN.matcher(symbols).matches()) {
+ if (TIME_UNIT_PATTERN.matcher(symbols).matches()) {
/*
* UCAR library has two methods for getting epoch: getDate() and getDateOrigin().
* The former adds to the origin the number that may appear before the unit, for example
@@ -317,6 +317,16 @@ final class VariableWrapper extends Variable {
}
/**
+ * Whether {@link #read()} invokes {@link Vector#compress(double)} on the returned vector.
+ *
+ * @return {@code false}.
+ */
+ @Override
+ protected boolean readTriesToCompress() {
+ return false;
+ }
+
+ /**
* Reads all the data for this variable and returns them as an array of a Java primitive type.
* Multi-dimensional variables are flattened as a one-dimensional array (wrapped in a vector).
* This method may cache the returned vector, at UCAR library choice.
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
index e456e86..8eb744a 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
@@ -273,7 +273,9 @@ public class NetcdfStoreProvider extends DataStoreProvider {
Object keepOpen;
final ChannelDataInput input = connector.getStorageAs(ChannelDataInput.class);
if (input != null) try {
- decoder = new ChannelDecoder(input, connector.getOption(OptionKey.ENCODING), geomlib, listeners);
+ final ChannelDecoder cd = new ChannelDecoder(input, connector.getOption(OptionKey.ENCODING), geomlib, listeners);
+ cd.applyOtherConventions();
+ decoder = cd;
keepOpen = input;
} catch (DataStoreException | ArithmeticException e) {
final String path = connector.getStorageAs(String.class);