You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sis.apache.org by de...@apache.org on 2021/08/31 17:25:17 UTC
[sis] 02/07: USe `X_DIMENSION` and `Y_DIMENSION` constants for 0
and 1. Fix a tile width which was sometime wrong (a `forceTileSize` flag
has been added).
This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit 2dfd4ad3295dfaf2f7a24e4a4c6458e0c87e96fa
Author: Martin Desruisseaux <ma...@geomatys.com>
AuthorDate: Tue Aug 31 15:23:52 2021 +0200
USe `X_DIMENSION` and `Y_DIMENSION` constants for 0 and 1.
Fix a tile width which was sometime wrong (a `forceTileSize` flag has been added).
---
.../apache/sis/internal/storage/inflater/ZIP.java | 3 +-
.../sis/storage/geotiff/CompressedSubset.java | 20 +++---
.../org/apache/sis/storage/geotiff/DataSubset.java | 12 ++--
.../sis/internal/storage/TiledGridCoverage.java | 82 +++++++++++++++-------
.../sis/internal/storage/TiledGridResource.java | 69 ++++++++++--------
5 files changed, 114 insertions(+), 72 deletions(-)
diff --git a/storage/sis-geotiff/src/main/java/org/apache/sis/internal/storage/inflater/ZIP.java b/storage/sis-geotiff/src/main/java/org/apache/sis/internal/storage/inflater/ZIP.java
index 156346b..f33b0cd 100644
--- a/storage/sis-geotiff/src/main/java/org/apache/sis/internal/storage/inflater/ZIP.java
+++ b/storage/sis-geotiff/src/main/java/org/apache/sis/internal/storage/inflater/ZIP.java
@@ -86,11 +86,10 @@ final class ZIP extends CompressionChannel {
if (++required >= input.buffer.capacity()) {
throw new BufferOverflowException();
}
- if (finished()) return -1;
input.ensureBufferContains(required);
setInput(input.buffer);
} else if (inflater.finished()) {
- break;
+ return -1;
} else {
throw new IOException();
}
diff --git a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/CompressedSubset.java b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/CompressedSubset.java
index b7ec4c0..2f2f948 100644
--- a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/CompressedSubset.java
+++ b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/CompressedSubset.java
@@ -109,13 +109,11 @@ final class CompressedSubset extends DataSubset {
* @param rasters potentially shared cache of rasters read by this {@code DataSubset}.
* @throws ArithmeticException if the number of tiles overflows 32 bits integer arithmetic.
*/
- CompressedSubset(final DataCube source, final TiledGridResource.Subset subset)
- throws DataStoreException
- {
+ CompressedSubset(final DataCube source, final TiledGridResource.Subset subset) throws DataStoreException {
super(source, subset);
- scanlineStride = multiplyFull(getTileSize(0), sourcePixelStride);
- final int between = sourcePixelStride * (getSubsampling(0) - 1);
- int afterLastBand = sourcePixelStride * (getTileSize(0) - 1);
+ scanlineStride = multiplyFull(getTileSize(X_DIMENSION), sourcePixelStride);
+ final int between = sourcePixelStride * (getSubsampling(X_DIMENSION) - 1);
+ int afterLastBand = sourcePixelStride * (getTileSize(X_DIMENSION) - 1);
if (includedBands != null && sourcePixelStride > 1) {
final int[] skips = new int[includedBands.length];
final int m = skips.length - 1;
@@ -197,12 +195,12 @@ final class CompressedSubset extends DataSubset {
final int[] subsampling, final Point location) throws IOException, DataStoreException
{
final DataType dataType = getDataType();
- final int width = pixelCount(lower, upper, subsampling, 0);
- final int height = pixelCount(lower, upper, subsampling, 1);
+ final int width = pixelCount(lower, upper, subsampling, X_DIMENSION);
+ final int height = pixelCount(lower, upper, subsampling, Y_DIMENSION);
final int chunksPerRow = width * (targetPixelStride / samplesPerChunk);
final int betweenRows = subsampling[1] - 1;
- final long head = beforeFirstBand + sourcePixelStride * (lower[0]);
- final long tail = afterLastBand - sourcePixelStride * (lower[0] + (width-1)*subsampling[0]);
+ final long head = beforeFirstBand + sourcePixelStride * (lower[X_DIMENSION]);
+ final long tail = afterLastBand - sourcePixelStride * (lower[X_DIMENSION] + (width-1)*subsampling[X_DIMENSION]);
/*
* `head` and `tail` are the number of sample values to skip at the beginning and end of each row.
* `betweenPixels` is the number of sample values to skip between each pixel, but the actual skips
@@ -219,7 +217,7 @@ final class CompressedSubset extends DataSubset {
assert (head % pixelsPerElement) == 0 : head;
if (inflater == null) {
inflater = Inflater.create(this, input(), source.getCompression(), source.getPredictor(),
- sourcePixelStride, getTileSize(0), chunksPerRow, samplesPerChunk, skipAfterChunks,
+ sourcePixelStride, getTileSize(X_DIMENSION), chunksPerRow, samplesPerChunk, skipAfterChunks,
pixelsPerElement, dataType);
}
final Inflater inflater = this.inflater;
diff --git a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
index 569a954..e0d4327 100644
--- a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
+++ b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
@@ -422,8 +422,8 @@ class DataSubset extends TiledGridCoverage implements Localized {
{
final DataType type = getDataType();
final int sampleSize = type.size(); // Assumed same as `SampleModel.getSampleSize(…)` by pre-conditions.
- final long width = subtractExact(upper[0], lower[0]);
- final long height = subtractExact(upper[1], lower[1]);
+ final long width = subtractExact(upper[X_DIMENSION], lower[X_DIMENSION]);
+ final long height = subtractExact(upper[Y_DIMENSION], lower[Y_DIMENSION]);
/*
* The number of bytes to read should not be greater than `byteCount`. It may be smaller however if only
* a subregion is read. Note that the `length` value may be different than `capacity` if the tile to read
@@ -431,15 +431,15 @@ class DataSubset extends TiledGridCoverage implements Localized {
* This length is used only for verification purpose so it does not need to be exact.
*/
final long length = ceilDiv(width * height * sourcePixelStride * sampleSize, Byte.SIZE);
- final long[] size = new long[] {multiplyFull(sourcePixelStride, getTileSize(0)), getTileSize(1)};
+ final long[] size = new long[] {multiplyFull(sourcePixelStride, getTileSize(X_DIMENSION)), getTileSize(Y_DIMENSION)};
/*
* If we use an interleaved sample model, each "element" from `HyperRectangleReader` perspective is actually a
* group of `sourcePixelStride` values. Note that in such case, we can not handle subsampling on the first axis.
* Such case should be handled by the `CompressedSubset` subclass instead, even if there is no compression.
*/
- assert sourcePixelStride == 1 || subsampling[0] == 1;
- lower[0] *= sourcePixelStride;
- upper[0] *= sourcePixelStride;
+ assert sourcePixelStride == 1 || subsampling[X_DIMENSION] == 1;
+ lower[X_DIMENSION] *= sourcePixelStride;
+ upper[X_DIMENSION] *= sourcePixelStride;
/*
* Read each plane ("banks" in Java2D terminology). Note that a single bank contains all bands
* in the interleaved sample model case. This block assumes that each bank element contains
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridCoverage.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridCoverage.java
index 8de9830..d6e9988 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridCoverage.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridCoverage.java
@@ -74,6 +74,12 @@ public abstract class TiledGridCoverage extends GridCoverage {
protected static final int BIDIMENSIONAL = 2;
/**
+ * The dimensions of <var>x</var> and <var>y</var> axes.
+ * Static constants for now, may become configurable fields in the future.
+ */
+ protected static final int X_DIMENSION = 0, Y_DIMENSION = 1;
+
+ /**
* The area to read in unit of the full coverage (without subsampling).
* This is the intersection between user-specified domain and the source
* {@link TiledGridResource} domain, expanded to an integer number of tiles.
@@ -81,6 +87,26 @@ public abstract class TiledGridCoverage extends GridCoverage {
private final GridExtent readExtent;
/**
+ * Whether to force the {@link #readExtent} tile intersection to the {@link #tileSize}.
+ * This is relevant only for the last column of tile matrix, because those tiles may be truncated
+ * if the image size is not a multiple of tile size. It is usually necessary to read those tiles
+ * fully anyway because otherwise, the pixels read from the storage would not be aligned with the
+ * pixels stored in the {@link WritableRaster}. However there is a few exceptions where the read
+ * extent should not be forced to the tile size:
+ *
+ * <ul>
+ * <li>If the image is untiled, then the {@link org.apache.sis.internal.storage.TiledGridResource.Subset}
+ * constructor assumes that only the requested region of the tile will be read.</li>
+ * <li>If the tile is truncated on the storage as well
+ * (note: this is rare. GeoTIFF for example always stores whole tiles).</li>
+ * </ul>
+ *
+ * <p>In current version this is a flag for the <var>x</var> dimension only. In a future version
+ * it could be flags for other dimensions as well (using bitmask) if it appears to be useful.</p>
+ */
+ private final boolean forceTileSize;
+
+ /**
* Size of all tiles in the domain of this {@code TiledGridCoverage}, without clipping and subsampling.
* All coverages created from the same {@link TiledGridResource} have the same tile size values.
* The length of this array is the number of dimensions in the source {@link GridExtent}.
@@ -152,10 +178,9 @@ public abstract class TiledGridCoverage extends GridCoverage {
private final Map<TiledGridResource.CacheKey, WritableRaster> rasters;
/**
- * The sample model for all rasters. The size of this sample model is the values of
- * the two first elements of {@link #tileSize} divided by subsampling after clipping.
- * If user requested to read only a subset of the bands, this sample model is already
- * the subset.
+ * The sample model for all rasters. The size of this sample model is the values of two elements
+ * of {@link #tileSize} divided by subsampling and clipped to the domain. If user requested to
+ * read only a subset of the bands, then this sample model is already the subset.
*/
protected final SampleModel model;
@@ -191,13 +216,13 @@ public abstract class TiledGridCoverage extends GridCoverage {
int tileStride = 1;
long indexOfFirstTile = 0;
for (int i=0; i<dimension; i++) {
- final int ts = tileSize[i];
- tmcOfFirstTile[i] = floorDiv(readExtent.getLow(i), ts);
- tileStrides [i] = tileStride;
- subSize [i] = (int) Math.min(((ts-1) / subsampling[i]) + 1, extent.getSize(i));
- indexOfFirstTile = addExact(indexOfFirstTile, multiplyExact(tmcOfFirstTile[i], tileStride));
- final int tileCount = toIntExact(ceilDiv(subset.sourceExtent.getSize(i), ts));
- tileStride = multiplyExact(tileCount, tileStride);
+ final int ts = tileSize[i];
+ tmcOfFirstTile[i] = floorDiv(readExtent.getLow(i), ts);
+ tileStrides[i] = tileStride;
+ subSize[i] = (int) Math.min(((ts-1) / subsampling[i]) + 1, extent.getSize(i));
+ indexOfFirstTile = addExact(indexOfFirstTile, multiplyExact(tmcOfFirstTile[i], tileStride));
+ int tileCount = toIntExact(ceilDiv(subset.sourceExtent.getSize(i), ts));
+ tileStride = multiplyExact(tileCount, tileStride);
}
this.indexOfFirstTile = toIntExact(indexOfFirstTile);
/*
@@ -206,12 +231,13 @@ public abstract class TiledGridCoverage extends GridCoverage {
* we want `ArithmeticException` to be thrown if the value is too high.
*/
SampleModel model = subset.modelForBandSubset;
- if (model.getWidth() != subSize[0] || model.getHeight() != subSize[1]) {
- model = model.createCompatibleSampleModel(subSize[0], subSize[1]);
+ if (model.getWidth() != subSize[X_DIMENSION] || model.getHeight() != subSize[Y_DIMENSION]) {
+ model = model.createCompatibleSampleModel(subSize[X_DIMENSION], subSize[Y_DIMENSION]);
}
this.model = model;
this.colors = subset.colorsForBandSubset;
this.fillValue = subset.fillValue;
+ forceTileSize = subSize[X_DIMENSION] * subsampling[X_DIMENSION] == tileSize[X_DIMENSION];
}
/**
@@ -400,7 +426,9 @@ public abstract class TiledGridCoverage extends GridCoverage {
* - Two-dimensional conversion from pixel coordinates to "real world" coordinates.
*/
final Map<String,Object> properties = DeferredProperty.forGridGeometry(getGridGeometry(), selectedDimensions);
- image = new TiledImage(properties, colors, imageSize[0], imageSize[1], tileLower[0], tileLower[1], result);
+ image = new TiledImage(properties, colors,
+ imageSize[X_DIMENSION], imageSize[Y_DIMENSION],
+ tileLower[X_DIMENSION], tileLower[Y_DIMENSION], result);
} catch (Exception e) { // Too many exception types for listing them all.
throw new CannotEvaluateException(Resources.forLocale(getLocale()).getString(
Resources.Keys.CanNotRenderImage_1, getDisplayName()), e);
@@ -477,7 +505,7 @@ public abstract class TiledGridCoverage extends GridCoverage {
* converts the `tileLower` coordinates to index in the `tileOffsets` and `tileByteCounts` vectors.
*/
indexInTileVector = indexOfFirstTile;
- int tileCountInQuery = 1;
+ int tileCountInQuery = 1;
for (int i=0; i<dimension; i++) {
final int lower = tileLower[i];
final int count = subtractExact(tileUpper[i], lower);
@@ -529,8 +557,8 @@ public abstract class TiledGridCoverage extends GridCoverage {
* Found a tile, but the sample model may be different because band order may be different.
* In both cases, we need to make sure that the raster starts at the expected coordinates.
*/
- final int x = getTileOrigin(0);
- final int y = getTileOrigin(1);
+ final int x = getTileOrigin(X_DIMENSION);
+ final int y = getTileOrigin(Y_DIMENSION);
if (!model.equals(tile.getSampleModel())) {
tile = WritableRaster.createWritableRaster(model, tile.getDataBuffer(), new Point(x, y));
} else if (tile.getMinX() != x || tile.getMinY() != y) {
@@ -632,12 +660,12 @@ public abstract class TiledGridCoverage extends GridCoverage {
* @param iterator the iterator for which to create a snapshot of its current position.
*/
public Snapshot(final AOI iterator) {
- coverage = iterator.getCoverage();
- this.tmcInSubset = iterator.tmcInSubset.clone();
- this.indexInResultArray = iterator.indexInResultArray;
- this.indexInTileVector = iterator.indexInTileVector;
- this.originX = iterator.getTileOrigin(0);
- this.originY = iterator.getTileOrigin(1);
+ coverage = iterator.getCoverage();
+ tmcInSubset = iterator.tmcInSubset.clone();
+ indexInResultArray = iterator.indexInResultArray;
+ indexInTileVector = iterator.indexInTileVector;
+ originX = iterator.getTileOrigin(X_DIMENSION);
+ originY = iterator.getTileOrigin(Y_DIMENSION);
}
/**
@@ -669,7 +697,7 @@ public abstract class TiledGridCoverage extends GridCoverage {
* only one (potentially big) tile, so the tile reading process become a reading of untiled data.
*/
long offset = subtractExact(coverage.readExtent.getLow(dimension), tileBase);
- upper[dimension] = Math.min(addExact(offset, coverage.readExtent.getSize(dimension)), tileSize);
+ long limit = Math.min(addExact(offset, coverage.readExtent.getSize(dimension)), tileSize);
if (offset < 0) {
/*
* Example: for `tileSize` = 10 pixels and `subsampling` = 3,
@@ -692,10 +720,14 @@ public abstract class TiledGridCoverage extends GridCoverage {
offset += s;
}
}
- if (offset >= upper[dimension]) {
+ if (offset >= limit) { // Test for intersection before we adjust the limit.
return false;
}
+ if (dimension == X_DIMENSION && coverage.forceTileSize) {
+ limit = tileSize;
+ }
lower[dimension] = offset;
+ upper[dimension] = limit;
}
return true;
}
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
index 9fdf1a2..1d6f1ea 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
@@ -40,6 +40,9 @@ import org.apache.sis.storage.event.StoreListeners;
import org.apache.sis.util.collection.WeakValueHashMap;
import org.apache.sis.util.ArraysExt;
+import static org.apache.sis.internal.storage.TiledGridCoverage.X_DIMENSION;
+import static org.apache.sis.internal.storage.TiledGridCoverage.Y_DIMENSION;
+
/**
* Base class of grid coverage resource storing data in tiles.
@@ -146,32 +149,19 @@ public abstract class TiledGridResource extends AbstractGridResource {
* This value is usually 1 because each sample value is usually stored in a separated element.
* However in multi-pixels packed sample model (e.g. bilevel image with 8 pixels per byte),
* it is difficult to start reading an image at <var>x</var> location other than a byte boundary.
- * By declaring an "atom" size of 8 sample values in dimension 0, the {@link Subset} constructor
+ * By declaring an "atom" size of 8 sample values in dimension X, the {@link Subset} constructor
* will ensure than the sub-region to read starts at a byte boundary when reading a bilevel image.
*
* <p>The default implementation returns the {@linkplain TiledGridCoverage#getPixelsPerElement()
- * number of pixels per data element} for dimension 0 and returns 1 for all other dimensions.</p>
+ * number of pixels per data element} for dimension X and returns 1 for all other dimensions.</p>
*
- * @param dimension the dimension for which to get the atom size.
- * This is in units of sample values (may be bits, bytes, floats, <i>etc</i>).
+ * @param xdim {@code true} for the size on <var>x</var> dimension, {@code false} for any other dimension.
* @return indivisible amount of sample values to read in the specified dimension. Must be ≥ 1.
+ * This is in units of sample values (may be bits, bytes, floats, <i>etc</i>).
* @throws DataStoreException if an error occurred while fetching the sample model.
*/
- protected int getAtomSize(final int dimension) throws DataStoreException {
- return (dimension == 0) ? TiledGridCoverage.getPixelsPerElement(getSampleModel()) : 1;
- }
-
- /**
- * Returns the maximal subsampling supported in the given dimension.
- * The default implementation puts no limit if {@code getAtomSize(dimension)} is 1,
- * and disables subsampling otherwise.
- *
- * @param dimension the dimension to test.
- * @return the maximal subsampling supported in the given dimension.
- * @throws DataStoreException if an error occurred while fetching the sample model.
- */
- protected int getMaximumSubsampling(final int dimension) throws DataStoreException {
- return getAtomSize(dimension) == 1 ? Integer.MAX_VALUE : 1;
+ protected int getAtomSize(final boolean xdim) throws DataStoreException {
+ return xdim ? TiledGridCoverage.getPixelsPerElement(getSampleModel()) : 1;
}
/**
@@ -309,7 +299,7 @@ public abstract class TiledGridResource extends AbstractGridResource {
/**
* The sample model for the bands to read (not the full set of bands in the resource).
- * The width is {@code tileSize[0]} and the height it {@code tileSize[1]},
+ * The width is {@code tileSize[X_DIMENSION]} and the height it {@code tileSize[Y_DIMENSION]},
* i.e. subsampling is <strong>not</strong> applied.
*/
final SampleModel modelForBandSubset;
@@ -364,13 +354,36 @@ public abstract class TiledGridResource extends AbstractGridResource {
* Note that it is possible to disable this restriction in a single dimension, typically the X one
* when reading a TIFF image using strips instead of tiles.
*/
- int tileWidth = tileSize[0];
- int tileHeight = tileSize[1];
- if (tileWidth >= sourceExtent.getSize(0)) {tileWidth = getAtomSize(0); sharedCache = false;}
- if (tileHeight >= sourceExtent.getSize(1)) {tileHeight = getAtomSize(1); sharedCache = false;}
- final GridDerivation target = gridGeometry.derive().chunkSize(tileWidth, tileHeight)
- .maximumSubsampling(getMaximumSubsampling(0), getMaximumSubsampling(1))
- .rounding(GridRoundingMode.ENCLOSING).subgrid(domain);
+ final int atomSizeX = getAtomSize(true);
+ final int atomSizeY = getAtomSize(false);
+ int tileWidth = tileSize[X_DIMENSION];
+ int tileHeight = tileSize[Y_DIMENSION];
+ if (tileWidth >= sourceExtent.getSize(X_DIMENSION)) {tileWidth = atomSizeX; sharedCache = false;}
+ if (tileHeight >= sourceExtent.getSize(Y_DIMENSION)) {tileHeight = atomSizeY; sharedCache = false;}
+ /*
+ * Note: if we allow X_DIMENSION and Y_DIMENSION to be anything in the future, then
+ * BIDIMENSIONAL must become `max(xDim, yDim) + 1` and array must be initialized to 1.
+ */
+ final int[] chunkSize = new int[TiledGridCoverage.BIDIMENSIONAL];
+ chunkSize[X_DIMENSION] = tileWidth;
+ chunkSize[Y_DIMENSION] = tileHeight;
+ /*
+ * Maximal subsampling supported. We put no restriction if subsamplig can occur anywhere
+ * ("atome size" of 1) and disable subsampling otherwise for avoiding code complexity.
+ */
+ final int[] maximumSubsampling = new int[chunkSize.length];
+ Arrays.fill(maximumSubsampling, Integer.MAX_VALUE);
+ if (atomSizeX != 1) maximumSubsampling[X_DIMENSION] = 1;
+ if (atomSizeY != 1) maximumSubsampling[Y_DIMENSION] = 1;
+ /*
+ * Build the domain in units of subsampled pixels, and get the same extent (`readExtent`)
+ * without subsampling, i.e. in units of cells of the original grid resource.
+ */
+ final GridDerivation target = gridGeometry.derive().chunkSize(chunkSize)
+ .maximumSubsampling(maximumSubsampling)
+ .rounding(GridRoundingMode.ENCLOSING)
+ .subgrid(domain);
+
domain = target.build();
readExtent = target.getIntersection();
subsampling = target.getSubsampling();
@@ -427,7 +440,7 @@ public abstract class TiledGridResource extends AbstractGridResource {
* @return whether the values to read on a row are contiguous.
*/
public boolean isXContiguous() {
- return includedBands == null && subsampling[0] == 1;
+ return includedBands == null && subsampling[X_DIMENSION] == 1;
}
}