You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sis.apache.org by de...@apache.org on 2018/11/06 19:24:30 UTC
[sis] 02/02: Partial fix for support of netCDF variables having an
unlimited dimension. https://issues.apache.org/jira/browse/SIS-439
This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit c1779efec550b420839dbca48147064b3be7dd31
Author: Martin Desruisseaux <ma...@geomatys.com>
AuthorDate: Tue Nov 6 20:23:41 2018 +0100
Partial fix for support of netCDF variables having an unlimited dimension.
https://issues.apache.org/jira/browse/SIS-439
---
.../java/org/apache/sis/image/PixelIterator.java | 8 +-
.../org/apache/sis/internal/util/Numerics.java | 34 ++++++
.../org/apache/sis/internal/util/NumericsTest.java | 15 +++
.../java/org/apache/sis/internal/netcdf/Axis.java | 2 +
.../org/apache/sis/internal/netcdf/DataType.java | 19 ++-
.../org/apache/sis/internal/netcdf/Variable.java | 13 +-
.../sis/internal/netcdf/impl/ChannelDecoder.java | 25 ++--
.../apache/sis/internal/netcdf/impl/Dimension.java | 26 ++--
.../sis/internal/netcdf/impl/VariableInfo.java | 135 +++++++++++++++++----
.../sis/internal/netcdf/impl/package-info.java | 1 +
.../sis/internal/netcdf/ucar/VariableWrapper.java | 8 ++
.../org/apache/sis/storage/netcdf/NetcdfStore.java | 2 +-
.../sis/storage/netcdf/NetcdfStoreProvider.java | 2 +-
.../org/apache/sis/internal/netcdf/TestCase.java | 6 +-
.../sis/internal/storage/io/ChannelDataInput.java | 16 ++-
.../internal/storage/io/HyperRectangleReader.java | 11 +-
.../org/apache/sis/internal/storage/io/Region.java | 65 ++++++----
17 files changed, 300 insertions(+), 88 deletions(-)
diff --git a/core/sis-raster/src/main/java/org/apache/sis/image/PixelIterator.java b/core/sis-raster/src/main/java/org/apache/sis/image/PixelIterator.java
index c076236..fc6642d 100644
--- a/core/sis-raster/src/main/java/org/apache/sis/image/PixelIterator.java
+++ b/core/sis-raster/src/main/java/org/apache/sis/image/PixelIterator.java
@@ -30,6 +30,7 @@ import org.apache.sis.util.resources.Errors;
import org.apache.sis.util.ArgumentChecks;
import static java.lang.Math.floorDiv;
+import static org.apache.sis.internal.util.Numerics.ceilDiv;
/**
@@ -170,13 +171,6 @@ public abstract class PixelIterator {
}
/**
- * Returns {@code numerator / denominator} rounded toward positive infinity.
- */
- private static int ceilDiv(final int numerator, final int denominator) {
- return -floorDiv(-numerator, denominator);
- }
-
- /**
* Computes the intersection between the given bounds and and {@code subArea} if {@code subArea} is non-null.
* If the result is empty, then the width and/or height are set to zero (not negative).
*/
diff --git a/core/sis-utility/src/main/java/org/apache/sis/internal/util/Numerics.java b/core/sis-utility/src/main/java/org/apache/sis/internal/util/Numerics.java
index 78b4ca5..519d83b 100644
--- a/core/sis-utility/src/main/java/org/apache/sis/internal/util/Numerics.java
+++ b/core/sis-utility/src/main/java/org/apache/sis/internal/util/Numerics.java
@@ -147,6 +147,40 @@ public final class Numerics extends Static {
}
/**
+ * Returns the smallest (closest to negative infinity) long value that is greater than or equals to x/y.
+ *
+ * @param x the dividend.
+ * @param y the divisor.
+ * @return x/y rounded toward positive infinity.
+ *
+ * @see Math#floorDiv(int, int)
+ */
+ public static int ceilDiv(final int x, final int y) {
+ int r = x / y;
+ if ((x ^ y) >= 0 && (r * y != x)) {
+ r++;
+ }
+ return r;
+ }
+
+ /**
+ * Returns the smallest (closest to negative infinity) long value that is greater than or equals to x/y.
+ *
+ * @param x the dividend.
+ * @param y the divisor.
+ * @return x/y rounded toward positive infinity.
+ *
+ * @see Math#floorDiv(long, long)
+ */
+ public static long ceilDiv(final long x, final long y) {
+ long r = x / y;
+ if ((x ^ y) >= 0 && (r * y != x)) {
+ r++;
+ }
+ return r;
+ }
+
+ /**
* If the given value is presents in the cache, returns the cached value.
* Otherwise returns the given value as-is.
*
diff --git a/core/sis-utility/src/test/java/org/apache/sis/internal/util/NumericsTest.java b/core/sis-utility/src/test/java/org/apache/sis/internal/util/NumericsTest.java
index bf2889f..dcaa21e 100644
--- a/core/sis-utility/src/test/java/org/apache/sis/internal/util/NumericsTest.java
+++ b/core/sis-utility/src/test/java/org/apache/sis/internal/util/NumericsTest.java
@@ -41,6 +41,21 @@ import static org.junit.Assert.*;
@SuppressWarnings("UnnecessaryBoxing")
public final strictfp class NumericsTest extends TestCase {
/**
+ * Tests {@link Numerics#ceilDiv(int, int)} and {@link Numerics#ceilDiv(long, long)}.
+ */
+ @Test
+ public void testCeilDiv() {
+ assertEquals( 4, ceilDiv( 12, 3 ));
+ assertEquals( 4L, ceilDiv( 12L, 3L));
+ assertEquals( 3, ceilDiv( 8, 3 ));
+ assertEquals( 3L, ceilDiv( 8L, 3L));
+ assertEquals(-4, ceilDiv(-12, 3 ));
+ assertEquals(-4L, ceilDiv(-12L, 3L));
+ assertEquals(-2, ceilDiv( -8, 3 ));
+ assertEquals(-2L, ceilDiv( -8L, 3L));
+ }
+
+ /**
* Tests the {@link Numerics#cached(Object)} method.
*/
@Test
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
index 5939443..b070054 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
@@ -42,6 +42,8 @@ import org.apache.sis.util.ArraysExt;
public final class Axis {
/**
* The attributes to use for fetching dimension (in ISO-19115 sense) information, or {@code null} if unknown.
+ * Example: {@code "geospatial_lat_min"}, {@code "geospatial_lat_resolution"}, {@code DimensionNameType.ROW}.
+ * This is used by {@link org.apache.sis.storage.netcdf.MetadataReader} for information purpose only.
*/
public final AttributeNames.Dimension attributeNames;
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DataType.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DataType.java
index b14f320..b86a8cc 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DataType.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DataType.java
@@ -37,7 +37,7 @@ import org.apache.sis.util.Numbers;
*
* @author Johann Sorel (Geomatys)
* @author Martin Desruisseaux (Geomatys)
- * @version 0.8
+ * @version 1.0
* @since 0.8
* @module
*/
@@ -162,6 +162,23 @@ public enum DataType {
}
/**
+ * Returns the number of bytes for this data type, or 0 if unknown.
+ *
+ * @return number of bytes for this data type, or 0 if unknown.
+ */
+ public final int size() {
+ switch (number) {
+ case Numbers.BYTE: return Byte.BYTES;
+ case Numbers.SHORT: return Short.BYTES;
+ case Numbers.INTEGER: // Same as float
+ case Numbers.FLOAT: return Float.BYTES;
+ case Numbers.LONG: // Same as double
+ case Numbers.DOUBLE: return Double.BYTES;
+ default: return 0;
+ }
+ }
+
+ /**
* Returns the signed or unsigned variant of this data type.
* If this data type does not have the requested variant, then this method returns {@code this}.
*
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
index 3a5d650..3e4f952 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
@@ -28,7 +28,7 @@ import org.apache.sis.storage.DataStoreException;
*
* @author Martin Desruisseaux (Geomatys)
* @author Johann Sorel (Geomatys)
- * @version 0.8
+ * @version 1.0
* @since 0.3
* @module
*/
@@ -91,6 +91,14 @@ public abstract class Variable extends NamedElement {
}
/**
+ * Returns whether this variable can grow. A variable is unlimited if at least one of its dimension is unlimited.
+ * In netCDF 3 classic format, only the first dimension can be unlimited.
+ *
+ * @return whether this variable can grow.
+ */
+ public abstract boolean isUnlimited();
+
+ /**
* Returns {@code true} if the given variable can be used for generating an image.
* This method checks for the following conditions:
*
@@ -238,6 +246,9 @@ public abstract class Variable extends NamedElement {
for (int i=shape.length; --i>=0;) {
buffer.append('[').append(Integer.toUnsignedLong(shape[i])).append(']');
}
+ if (isUnlimited()) {
+ buffer.append(" (unlimited)");
+ }
return buffer.toString();
}
}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
index e63d5fa..1fa3e4e 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
@@ -69,7 +69,7 @@ import ucar.nc2.constants.CF;
*
* @author Johann Sorel (Geomatys)
* @author Martin Desruisseaux (Geomatys)
- * @version 0.8
+ * @version 1.0
*
* @see <a href="http://portal.opengeospatial.org/files/?artifact_id=43734">NetCDF Classic and 64-bit Offset Format (1.0)</a>
*
@@ -214,7 +214,7 @@ public final class ChannelDecoder extends Decoder {
* <ul>
* <li>Magic number: 'C','D','F'</li>
* <li>Version number: 1 or 2</li>
- * <li>Number of records</li>
+ * <li>Number of records | {@value #STREAMING}</li>
* <li>List of netCDF dimensions (see {@link #readDimensions(int)})</li>
* <li>List of global attributes (see {@link #readAttributes(int)})</li>
* <li>List of variables (see {@link #readVariables(int, Dimension[])})</li>
@@ -226,6 +226,7 @@ public final class ChannelDecoder extends Decoder {
* @param listeners where to send the warnings.
* @throws IOException if an error occurred while reading the channel.
* @throws DataStoreException if the content of the given channel is not a netCDF file.
+ * @throws ArithmeticException if a variable is too large.
*/
public ChannelDecoder(final ChannelDataInput input, final Charset encoding, final GeometryLibrary geomlib,
final WarningListeners<DataStore> listeners) throws IOException, DataStoreException
@@ -277,9 +278,9 @@ public final class ChannelDecoder extends Decoder {
}
}
}
- attributeMap = attributes;
- this.variables = variables;
- variableMap = NamedElement.toCaseInsensitiveNameMap(variables, NAME_LOCALE);
+ this.attributeMap = attributes;
+ this.variables = variables;
+ this.variableMap = NamedElement.toCaseInsensitiveNameMap(variables, NAME_LOCALE);
}
/**
@@ -470,13 +471,14 @@ public final class ChannelDecoder extends Decoder {
for (int i=0; i<nelems; i++) {
final String name = readName();
int length = input.readInt();
- if (length == 0) {
+ boolean isUnlimited = (length == 0);
+ if (isUnlimited) {
length = numrecs;
if (length == STREAMING) {
throw new DataStoreContentException(errors().getString(Errors.Keys.MissingValueForProperty_1, "numrecs"));
}
}
- dimensions[i] = new Dimension(name, length);
+ dimensions[i] = new Dimension(name, length, isUnlimited);
}
dimensionMap = Dimension.toCaseInsensitiveNameMap(dimensions, NAME_LOCALE);
return dimensions;
@@ -565,8 +567,15 @@ public final class ChannelDecoder extends Decoder {
}
}
variables[j] = new VariableInfo(input, name, varDims, attributes,
- DataType.valueOf(input.readInt()), input.readInt(), readOffset());
+ DataType.valueOf(input.readInt()), input.readInt(), readOffset(), listeners);
}
+ /*
+ * The VariableInfo constructor determined if the variables are "unlimited" or not.
+ * The number of unlimited variable determines to padding to apply, because of an
+ * historical particularity in netCDF format. Those final adjustment can be done
+ * only after we finished creating all variables.
+ */
+ VariableInfo.complete(variables);
return variables;
}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/Dimension.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/Dimension.java
index 2851128..32ffd2b 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/Dimension.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/Dimension.java
@@ -26,7 +26,7 @@ import org.apache.sis.internal.netcdf.NamedElement;
*
* @author Johann Sorel (Geomatys)
* @author Martin Desruisseaux (Geomatys)
- * @version 0.8
+ * @version 1.0
* @since 0.3
* @module
*/
@@ -42,14 +42,22 @@ final class Dimension extends NamedElement {
final int length;
/**
+ * Whether this dimension is the "record" (also known as "unlimited") dimension.
+ * There is at most one record dimension in a well-formed netCDF file.
+ */
+ final boolean isUnlimited;
+
+ /**
* Creates a new dimension of the given name and length.
*
- * @param name the dimension name.
- * @param length the number of grid cell value along this dimension, as an unsigned number.
+ * @param name the dimension name.
+ * @param length the number of grid cell value along this dimension, as an unsigned number.
+ * @param isUnlimited whether this dimension is the "record" (also known as "unlimited") dimension.
*/
- Dimension(final String name, final int length) {
- this.name = name;
- this.length = length;
+ Dimension(final String name, final int length, final boolean isUnlimited) {
+ this.name = name;
+ this.length = length;
+ this.isUnlimited = isUnlimited;
}
/**
@@ -72,6 +80,10 @@ final class Dimension extends NamedElement {
*/
@Override
public String toString() {
- return name + '[' + length() + ']';
+ final StringBuilder buffer = new StringBuilder(name).append('[').append(length()).append(']');
+ if (isUnlimited) {
+ buffer.append(" (unlimited)");
+ }
+ return buffer.toString();
}
}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
index 70ca4ce..22cfc32 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
@@ -32,9 +32,9 @@ import org.apache.sis.internal.storage.io.Region;
import org.apache.sis.storage.DataStoreException;
import org.apache.sis.storage.DataStoreContentException;
import org.apache.sis.storage.netcdf.AttributeNames;
+import org.apache.sis.util.logging.WarningListeners;
import org.apache.sis.util.resources.Errors;
import org.apache.sis.util.CharSequences;
-import org.apache.sis.util.Numbers;
import org.apache.sis.math.Vector;
@@ -45,7 +45,7 @@ import org.apache.sis.math.Vector;
*
* @author Johann Sorel (Geomatys)
* @author Martin Desruisseaux (Geomatys)
- * @version 0.8
+ * @version 1.0
* @since 0.3
* @module
*/
@@ -78,13 +78,32 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
private final String name;
/**
- * The dimensions of this variable, in order. When iterating over the values stored in this variable
- * (a flattened one-dimensional sequence of values), index in the domain of {@code dimensions[0]}
- * varies faster, followed by index in the domain of {@code dimensions[1]}, <i>etc.</i>
+ * The dimensions of this variable, in the order they appear in netCDF file. When iterating over the values stored in
+ * this variable (a flattened one-dimensional sequence of values), index in the domain of {@code dimensions[length-1]}
+ * varies faster, followed by index in the domain of {@code dimensions[length-2]}, <i>etc.</i>
*/
final Dimension[] dimensions;
/**
+ * The offset (in bytes) to apply for moving to the next record in variable data, or -1 if unknown.
+ * A record is a <var>n-1</var> dimensional cube spanning all dimensions except the unlimited one.
+ * This concept is relevant only for {@linkplain #isUnlimited() unlimited variables}.
+ * For other kind of variables, this field is ignored and its value can be arbitrary.
+ *
+ * <p>On construction, this field is initialized to the number of bytes in this variable, omitting
+ * the unlimited dimension (if any), or 0 if unknown. After {@link #complete(VariableInfo[])} has
+ * been invoked, this field is modified as below:</p>
+ *
+ * <ul>
+ * <li>For {@linkplain #isUnlimited() unlimited variables}, this is the number of bytes to
+ * skip <strong>after</strong> the data has been read for reading the next record.</li>
+ * <li>For other variables, this field is ignored. In current implementation it is left to
+ * the size of this variable, and kept for information purpose only.</li>
+ * </ul>
+ */
+ private long offsetToNextRecord;
+
+ /**
* The attributes associates to the variable, or an empty map if none.
* Values can be:
*
@@ -148,8 +167,9 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
* @param dimensions the dimensions of this variable.
* @param attributes the attributes associates to the variable, or an empty map if none.
* @param dataType the netCDF type of data, or {@code null} if unknown.
- * @param size the variable size, used for verification purpose only.
+ * @param size the variable size. May be inaccurate and ignored.
* @param offset the offset where the variable data begins in the netCDF file.
+ * @param warnings where to report warnings, if any.
*/
VariableInfo(final ChannelDataInput input,
final String name,
@@ -157,7 +177,8 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
final Map<String,Object> attributes,
DataType dataType,
final int size,
- final long offset) throws DataStoreException
+ final long offset,
+ final WarningListeners<?> warnings) throws DataStoreException
{
final Object isUnsigned = attributes.get(CDM.UNSIGNED);
if (isUnsigned != null) {
@@ -170,11 +191,31 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
/*
* The 'size' value is provided in the netCDF files, but doesn't need to be stored since it
* is redundant with the dimension lengths and is not large enough for big variables anyway.
+ * Instead we compute the length ourselves, excluding the unlimited dimension.
+ */
+ if (dataType != null) {
+ offsetToNextRecord = dataType.size(); // May be zero if unknown data type.
+ for (int i=0; i<dimensions.length; i++) {
+ final Dimension dim = dimensions[i];
+ if (!dim.isUnlimited) {
+ offsetToNextRecord = Math.multiplyExact(offsetToNextRecord, dim.length());
+ } else if (i != 0) {
+ // Unlimited dimension, if any, must be first in a netCDF 3 classic format.
+ throw new DataStoreException(warnings.getLocale(), "netCDF", input.filename, null);
+ }
+ }
+ }
+ /*
+ * Prepare the object to be used for reading data cube efficiently from the input channel.
+ * Note that 'dataType' can not be null if 'offsetToNextRecord' is non-zero.
*/
- if (dataType != null && dataType.number >= Numbers.BYTE && dataType.number <= Numbers.DOUBLE) {
+ if (offsetToNextRecord != 0) {
reader = new HyperRectangleReader(dataType.number, input, offset);
} else {
reader = null;
+ if (size != -1) { // Maximal unsigned value, means possible overflow.
+ offsetToNextRecord = Integer.toUnsignedLong(size);
+ }
}
/*
* If the "_CoordinateAliasForDimension" attribute is defined, then its value will be used
@@ -210,6 +251,42 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
}
/**
+ * Performs the final adjustment of the {@link #offsetToNextRecord} field of all the given variables.
+ * This method applies padding except for the special case documented in netCDF specification:
+ * <cite>"In the special case when there is only one {@linkplain #isUnlimited() record variable}
+ * and it is of type character, byte, or short, no padding is used between record slabs,
+ * so records after the first record do not necessarily start on four-byte boundaries"</cite>.
+ *
+ * <p>After padding has been applied, this method set the {@link #offsetToNextRecord} of all unlimited
+ * variables to the number of bytes to skip before reading the next record.</p>
+ */
+ static void complete(final VariableInfo[] variables) {
+ final VariableInfo[] unlimited = new VariableInfo[variables.length];
+ int count = 0; // Number of valid elements in the 'unlimited' array.
+ long total = 0; // Sum of the size of all variables having a unlimited dimension.
+ boolean isUnknown = false; // True if 'total' is actually unknown.
+ for (final VariableInfo variable : variables) {
+ if (variable.isUnlimited()) {
+ // Apply 4 bytes padding.
+ final long offsetToNextRecord = Math.addExact(variable.offsetToNextRecord, Integer.BYTES - 1) & ~(Integer.BYTES - 1);
+ unlimited[count++] = variable;
+ isUnknown |= (offsetToNextRecord == 0);
+ total = Math.addExact(total, offsetToNextRecord);
+ }
+ }
+ if (isUnknown) {
+ // If at least one unlimited variable has unknown size, the whole record stride is unknown.
+ for (int i=0; i<count; i++) {
+ unlimited[i].offsetToNextRecord = -1;
+ }
+ } else if (count == 1) {
+ unlimited[0].offsetToNextRecord = 0; // Special case cited in method javadoc.
+ } else for (int i=0; i<count; i++) {
+ unlimited[i].offsetToNextRecord = total - unlimited[i].offsetToNextRecord;
+ }
+ }
+
+ /**
* Returns the name of this variable.
*/
@Override
@@ -260,6 +337,16 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
}
/**
+ * Returns whether this variable can grow. A variable is unlimited if at least one of its dimension is unlimited.
+ * In netCDF 3 classic format, only the first dimension can be unlimited.
+ */
+ @Override
+ public boolean isUnlimited() {
+ // The constructor verified that only the first dimension is unlimited.
+ return (dimensions.length != 0) && dimensions[0].isUnlimited;
+ }
+
+ /**
* Returns {@code true} if this variable seems to be a coordinate system axis,
* determined by comparing its name with the name of all dimensions in the netCDF file.
*/
@@ -416,26 +503,22 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
if (reader == null) {
throw new DataStoreContentException(unknownType());
}
- long length = 1;
- boolean overflow = false;
- for (final Dimension dimension : dimensions) {
- length *= dimension.length();
- if (length > Integer.MAX_VALUE) {
- overflow = true;
- }
- }
- if (overflow) {
- throw new DataStoreContentException(Errors.format(Errors.Keys.ExcessiveListSize_2, name, length));
- }
- final int dimension = dimensions.length;
- final long[] size = new long[dimension];
- final int [] sub = new int [dimension];
+ final int dimension = dimensions.length;
+ final long[] regionLower = new long[dimension];
+ final long[] regionUpper = new long[dimension];
+ final int [] subsamplings = new int [dimension];
for (int i=0; i<dimension; i++) {
- sub [i] = 1;
- size[i] = dimensions[(dimension - 1) - i].length();
+ regionUpper [i] = dimensions[(dimension - 1) - i].length();
+ subsamplings[i] = 1;
+ }
+ final Region region = new Region(regionUpper, regionLower, regionUpper, subsamplings);
+ if (isUnlimited()) {
+ if (offsetToNextRecord < 0) {
+ throw new DataStoreException(Errors.format(Errors.Keys.CanNotRead_1, name));
+ }
+ region.skipAfterLastDimension(offsetToNextRecord / dataType.size());
}
- values = Vector.create(reader.read(new Region(size, new long[dimension], size, sub)), dataType.isUnsigned)
- .compress(0);
+ values = Vector.create(reader.read(region), dataType.isUnsigned).compress(0);
}
return values;
}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/package-info.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/package-info.java
index 3a34638..e06d20d 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/package-info.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/package-info.java
@@ -25,6 +25,7 @@
* <ul>
* <li><a href="http://www.opengeospatial.org/standards/netcdf">NetCDF standards on OGC web site</a></li>
* <li><a href="http://portal.opengeospatial.org/files/?artifact_id=43734">NetCDF Classic and 64-bit Offset Format (1.0)</a></li>
+ * <li><a href="https://www.unidata.ucar.edu/software/netcdf/docs/file_format_specifications.html">NetCDF on UCAR web site.</a></li>
* </ul>
*
* @author Johann Sorel (Geomatys)
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
index b32a352..056124a 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
@@ -118,6 +118,14 @@ final class VariableWrapper extends Variable {
}
/**
+ * Returns whether this variable can grow. A variable is unlimited if at least one of its dimension is unlimited.
+ */
+ @Override
+ public boolean isUnlimited() {
+ return variable.isUnlimited();
+ }
+
+ /**
* Returns {@code true} if this variable seems to be a coordinate system axis.
*/
@Override
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStore.java b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStore.java
index 3629377..9af51ca 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStore.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStore.java
@@ -99,7 +99,7 @@ public class NetcdfStore extends DataStore implements Aggregate {
location = connector.getStorageAs(URI.class);
try {
decoder = NetcdfStoreProvider.decoder(listeners, connector);
- } catch (IOException e) {
+ } catch (IOException | ArithmeticException e) {
throw new DataStoreException(e);
}
if (decoder == null) {
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
index cd8c5bd..e456e86 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
@@ -275,7 +275,7 @@ public class NetcdfStoreProvider extends DataStoreProvider {
if (input != null) try {
decoder = new ChannelDecoder(input, connector.getOption(OptionKey.ENCODING), geomlib, listeners);
keepOpen = input;
- } catch (DataStoreException e) {
+ } catch (DataStoreException | ArithmeticException e) {
final String path = connector.getStorageAs(String.class);
if (path != null) try {
decoder = createByReflection(path, false, geomlib, listeners);
diff --git a/storage/sis-netcdf/src/test/java/org/apache/sis/internal/netcdf/TestCase.java b/storage/sis-netcdf/src/test/java/org/apache/sis/internal/netcdf/TestCase.java
index bf9e281..03497f3 100644
--- a/storage/sis-netcdf/src/test/java/org/apache/sis/internal/netcdf/TestCase.java
+++ b/storage/sis-netcdf/src/test/java/org/apache/sis/internal/netcdf/TestCase.java
@@ -37,7 +37,8 @@ import static org.junit.Assert.*;
/**
- * Base class of netCDF tests. Subclasses shall override the {@link #createDecoder(TestData)}.
+ * Base class of netCDF tests. The base class uses the UCAR decoder, which is taken as a reference implementation.
+ * Subclasses testing Apache SIS implementation needs to override the {@link #createDecoder(TestData)}.
*
* <p>This class is <strong>not</strong> thread safe - do not run subclasses in parallel.</p>
*
@@ -60,6 +61,9 @@ public abstract strictfp class TestCase extends org.apache.sis.test.TestCase {
/**
* The decoders cached by {@link #selectDataset(TestData)}.
+ * The make is non-empty only during the execution of a single test class.
+ *
+ * @see #closeAllDecoders()
*/
private static final Map<TestData,Decoder> DECODERS = new EnumMap<>(TestData.class);
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/ChannelDataInput.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/ChannelDataInput.java
index db4e0ca..6e33098 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/ChannelDataInput.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/ChannelDataInput.java
@@ -823,15 +823,23 @@ public class ChannelDataInput extends ChannelData {
* @return the string decoded from the {@code length} next bytes.
* @throws IOException if an error occurred while reading the bytes, or if the given encoding is invalid.
*/
- public final String readString(final int length, final Charset encoding) throws IOException {
+ public final String readString(int length, final Charset encoding) throws IOException {
+ final byte[] array;
+ int position;
if (buffer.hasArray() && length <= buffer.capacity()) {
ensureBufferContains(length);
- final int position = buffer.position(); // Must be after 'ensureBufferContains(int)'.
+ position = buffer.position(); // Must be after 'ensureBufferContains(int)'.
buffer.position(position + length); // Before 'new String' for consistency with the 'else' block in case of UnsupportedEncodingException.
- return new String(buffer.array(), buffer.arrayOffset() + position, length, encoding);
+ array = buffer.array();
+ position += buffer.arrayOffset();
} else {
- return new String(readBytes(length), encoding);
+ array = readBytes(length);
+ position = 0;
}
+ while (length > 0 && array[position + (length - 1)] == 0) {
+ length--; // Skip trailing 0 (end of string in C/C++ languages).
+ }
+ return new String(array, position, length, encoding);
}
/**
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
index e2ddb66..e2b0859 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
@@ -33,7 +33,7 @@ import org.apache.sis.util.Debug;
*
* @author Johann Sorel (Geomatys)
* @author Martin Desruisseaux (Geomatys)
- * @version 0.8
+ * @version 1.0
* @since 0.7
* @module
*/
@@ -116,14 +116,15 @@ public final class HyperRectangleReader {
* @throws IOException if an error occurred while transferring data from the channel.
*/
public Object read(final Region region) throws IOException {
- final int contiguousDataLength = region.targetLength(region.contiguousDataDimension);
- final long[] strides = new long[region.getDimension() - region.contiguousDataDimension];
+ final int contiguousDataDimension = region.contiguousDataDimension();
+ final int contiguousDataLength = region.targetLength(contiguousDataDimension);
+ final long[] strides = new long[region.getDimension() - contiguousDataDimension];
final int[] cursor = new int[strides.length];
final int sizeShift = reader.dataSizeShift();
long streamPosition = origin + (region.startAt << sizeShift);
int arrayPosition = 0;
for (int i=0; i<strides.length; i++) {
- strides[i] = (region.skips[i + region.contiguousDataDimension] + contiguousDataLength) << sizeShift;
+ strides[i] = (region.skips[i + contiguousDataDimension] + contiguousDataLength) << sizeShift;
assert (strides[i] > 0) : i;
}
try {
@@ -141,7 +142,7 @@ loop: do {
* new row, or a new plane, or a new cube?). This determine how many bytes we have to
* skip.
*/
- if (++cursor[i] < region.targetSize[region.contiguousDataDimension + i]) {
+ if (++cursor[i] < region.targetSize[contiguousDataDimension + i]) {
streamPosition += strides[i];
arrayPosition += contiguousDataLength;
continue loop;
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
index 345aeb3..486d425 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
@@ -16,6 +16,8 @@
*/
package org.apache.sis.internal.storage.io;
+import org.apache.sis.internal.util.Numerics;
+
/**
* A sub-area in a <var>n</var>-dimensional hyper-rectangle, optionally with sub-sampling.
@@ -29,17 +31,12 @@ package org.apache.sis.internal.storage.io;
*
* @author Johann Sorel (Geomatys)
* @author Martin Desruisseaux (Geomatys)
- * @version 0.7
+ * @version 1.0
* @since 0.7
* @module
*/
public final class Region {
/**
- * Total length of the sequence of values, ignoring sub-area and sub-sampling.
- */
- final long sourceLength;
-
- /**
* The size after reading only the sub-region at the given sub-sampling.
* The length of this array is the hyper-rectangle dimension.
*
@@ -54,7 +51,7 @@ public final class Region {
final long startAt;
/**
- * Number of values to skip after having read a values.
+ * Number of values to skip after having read values.
*
* <ol>
* <li>{@code skips[0]} is the number of values to skip after each single value on the same line.</li>
@@ -69,12 +66,6 @@ public final class Region {
final long[] skips;
/**
- * Number of dimensions for which we can collapse the read operations in a single operation because their
- * data are contiguous. This is the index of the first non-zero element in the {@link #skips} array.
- */
- final int contiguousDataDimension;
-
- /**
* Creates a new region. It is caller's responsibility to ensure that:
* <ul>
* <li>all arrays have the same length</li>
@@ -82,6 +73,7 @@ public final class Region {
* <li>{@code regionLower[i] >= 0} for all <var>i</var></li>
* <li>{@code regionLower[i] < regionUpper[i] <= size[i]} for all <var>i</var></li>
* <li>{@code subsamplings[i] > 0} for all <var>i</var></li>
+ * <li>The total length of data to read does not exceed {@link Integer#MAX_VALUE}.</li>
* </ul>
*
* @param size the number of elements along each dimension.
@@ -97,27 +89,35 @@ public final class Region {
long stride = 1;
long skip = 0;
for (int i=0; i<dimension;) {
- final int step = subsamplings[i];
- final long lower = regionLower [i];
- final long count = (regionUpper [i] - lower + (step-1)) / step; // (upper-lower)/step rounded toward up, provided all values are positive.
+ final int step = subsamplings[i];
+ final long lower = regionLower[i];
+ final long count = Numerics.ceilDiv(regionUpper[i] - lower, step);
final long upper = lower + ((count-1) * step + 1);
final long span = size[i];
assert (count > 0) && (lower >= 0) && (upper > lower) && (upper <= span) : i;
targetSize[i] = Math.toIntExact(count);
- position += stride * lower;
- skip += stride * (span - (upper - lower));
- skips[i] += stride * (step - 1);
- stride *= span;
+ position = Math.addExact(position, Math.multiplyExact(stride, lower));
+ skip = Math.addExact(skip, Math.multiplyExact(stride, span - (upper - lower)));
+ skips[i] = Math.addExact(skips[i], Math.multiplyExact(stride, step - 1));
+ stride = Math.multiplyExact(stride, span);
skips[++i] = skip;
}
startAt = position;
- sourceLength = stride;
- int i;
- for (i=0; i<dimension; i++) {
- if (skips[i] != 0) break;
- }
- contiguousDataDimension = i;
+ }
+
+ /**
+ * Increases the number of bytes that need to be skipped before incrementing the index
+ * in the last dimension of the hyper-cube. Current implementation allows to alter the
+ * reading for the last dimension only, because this is the only dimension needed for
+ * supporting netCDF "unlimited" dimension. Future versions may expand to other dimensions
+ * if needed.
+ *
+ * @param n number of bytes to skip.
+ */
+ public void skipAfterLastDimension(final long n) {
+ final int i = skips.length - 2; // Reminder: skips.length == dimension + 1.
+ skips[i] = Math.addExact(skips[i], n);
}
/**
@@ -130,6 +130,19 @@ public final class Region {
}
/**
+ * Number of dimensions for which we can collapse the read operations in a single operation because their
+ * data are contiguous. This is the index of the first non-zero element in the {@link #skips} array.
+ */
+ final int contiguousDataDimension() {
+ final int dimension = skips.length - 1;
+ int i;
+ for (i=0; i<dimension; i++) {
+ if (skips[i] != 0) break;
+ }
+ return i;
+ }
+
+ /**
* Returns the total number of values to be read from the sub-region while applying the sub-sampling.
* This method takes in account only the given number of dimensions.
*/