You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sis.apache.org by de...@apache.org on 2020/10/19 14:47:26 UTC
[sis] 03/04: Fix offset and stride calculation in netCDF files
having an unlimited dimension.
This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit 43d7cba7bb34d7a58b0b0ff7b2b9ebd7ce0aca2b
Author: Martin Desruisseaux <ma...@geomatys.com>
AuthorDate: Mon Oct 19 15:47:41 2020 +0200
Fix offset and stride calculation in netCDF files having an unlimited dimension.
---
.../sis/internal/netcdf/impl/VariableInfo.java | 10 +++---
.../internal/storage/io/HyperRectangleReader.java | 2 +-
.../org/apache/sis/internal/storage/io/Region.java | 39 ++++++++++++++++++----
3 files changed, 38 insertions(+), 13 deletions(-)
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
index ab58f0f..89de616 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
@@ -192,8 +192,8 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
}
this.dataType = dataType;
/*
- * The 'size' value is provided in the netCDF files, but doesn't need to be stored since it
- * is redundant with the dimension lengths and is not large enough for big variables anyway.
+ * The `size` value is provided in the netCDF files, but does not need to be stored because it can
+ * be computed from dimension lengths and its type is not wide enough for large variables anyway.
* Instead we compute the length ourselves, excluding the unlimited dimension.
*/
if (dataType != null && (offsetToNextRecord = dataType.size()) != 0) {
@@ -276,9 +276,9 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
static void complete(final VariableInfo[] variables) {
final Set<CharSequence> referencedAsAxis = new HashSet<>();
final VariableInfo[] unlimited = new VariableInfo[variables.length];
- int count = 0; // Number of valid elements in the 'unlimited' array.
+ int count = 0; // Number of valid elements in the `unlimited` array.
long recordStride = 0; // Sum of the size of all variables having a unlimited dimension.
- boolean isUnknown = false; // True if 'total' is actually unknown.
+ boolean isUnknown = false; // True if `total` is actually unknown.
for (final VariableInfo variable : variables) {
// Opportunistically store names of all axes listed in "coordinates" attributes of all variables.
referencedAsAxis.addAll(Arrays.asList(variable.getCoordinateVariables()));
@@ -594,7 +594,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
replaceNaN(array);
/*
* If we can convert a double[] array to a float[] array, we should do that before
- * to invoke 'setValues(array)' - we can not rely on data.compress(tolerance). The
+ * to invoke `setValues(array)` - we can not rely on data.compress(tolerance). The
* reason is because we assume that float[] arrays are accurate in base 10 even if
* the data were originally stored as doubles. The Vector class does not make such
* assumption since it is specific to what we observe with netCDF files. To enable
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
index 8f1aeb5..e0d2041 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
@@ -128,7 +128,7 @@ public final class HyperRectangleReader {
final long[] strides = new long[region.getDimension() - contiguousDataDimension];
final int[] cursor = new int[strides.length];
final int sampleSize = sampleSize();
- long streamPosition = Math.addExact(origin, Math.multiplyExact(region.startAt, sampleSize));
+ long streamPosition = Math.addExact(origin, region.offset(sampleSize));
int arrayPosition = 0;
for (int i=0; i<strides.length; i++) {
strides[i] = region.stride(i + contiguousDataDimension, contiguousDataLength, sampleSize);
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
index 25d0be3..c6f817c 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
@@ -69,6 +69,7 @@ public final class Region {
* <li><i>etc.</i></li>
* </ol>
*
+ * Values are determined by the {@code subsamplings} argument given at construction time.
* The length of this array is the hyper-rectangle dimension plus one.
*/
private final long[] skips;
@@ -128,11 +129,12 @@ public final class Region {
* The strides are computed automatically at construction time, but this method can be invoked in some rare cases
* where those values need to be modified (example: for adapting to the layout of netCDF "unlimited" variable).
*
- * <div class="note"><b>Example:</b> in a cube of dimension 10×10×10, the number of values between indices
- * (0,0,1) and (0,0,2) is 100. If the values type is {@code float}, invoking {@code increaseStride(1, 12)}
- * will increase this value to 103 (computed as 100 + 12/{@value Float#SIZE}). {@link HyperRectangleReader}
- * will still read only the requested 100 values, but will skip 3 more values when moving from plane 1 to
- * plane 2.</div>
+ * <div class="note"><b>Example:</b>
+ * in a cube of dimension 10×10×10, the number of values between indices (0,0,1) and (0,0,2) is 100.
+ * If the values type is {@code float}, invoking {@code setAdditionalByteOffset(1, 12)} will increase
+ * this value to 103 (computed as 100 + 12/{@value Float#BYTES}).
+ * {@link HyperRectangleReader} will still read only the requested 100 values,
+ * but will skip 3 more values when moving from plane 1 to plane 2.</div>
*
* This method is the only one in this {@link Region} class to use a count of bytes instead than a count
* of sample values.
@@ -167,13 +169,36 @@ public final class Region {
for (i=0; i<dimension; i++) {
if (skips[i] != 0) break;
}
+ if (skipBytes != null) {
+ final int n = Math.min(i, skipBytes.length);
+ for (int j=0; j<n; j++) {
+ if (skipBytes[j] != 0) {
+ return j;
+ }
+ }
+ }
return i;
}
/**
- * Computes the number of stride for the given dimension. Caller must provide a base stride value,
+ * Returns the offset in bytes where reading should start.
+ * Offset is relative to the first sample value of the hyper-cube.
+ *
+ * @param sampleSize size of sample values, in bytes.
+ * @return offset in bytes relative to the first sample value.
+ */
+ final long offset(long sampleSize) {
+ if (skipBytes != null) {
+ // This additional offset is in bytes.
+ sampleSize = addExact(sampleSize, skipBytes[0]);
+ }
+ return Math.multiplyExact(startAt, sampleSize);
+ }
+
+ /**
+ * Computes stride in number of bytes for the given dimension. Caller must provide a base stride value,
* which is the stride that we would have in absence of subregion and additional bytes to skip.
- * This method adds the given base the number of bytes to skip.
+ * This method adds to that given base the number of bytes to skip.
*
* @param dimension the dimension for which to compute the stride.
* @param base stride that we would have in absence of subregion and bytes to skip.