You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sis.apache.org by de...@apache.org on 2021/07/29 17:08:36 UTC
[sis] branch geoapi-4.0 updated (d53b8a5 -> b71d71d)
This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a change to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git.
from d53b8a5 Take in account the band subset specified by the `range` argument in a call to `GridCoverage.render(…)` on a GeoTIFF image.
new 54eb82c Expand the support to all primitive types.
new b71d71d Use direct buffer by default instead of heap buffer for I/O operations.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/sis/internal/geotiff/Uncompressed.java | 228 +++++++++++++++++++--
.../org/apache/sis/storage/StorageConnector.java | 39 ++--
2 files changed, 235 insertions(+), 32 deletions(-)
[sis] 02/02: Use direct buffer by default instead of heap buffer
for I/O operations.
Posted by de...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit b71d71d549ed207d5cae618298bedfa948daa216
Author: Martin Desruisseaux <ma...@geomatys.com>
AuthorDate: Thu Jul 29 19:07:41 2021 +0200
Use direct buffer by default instead of heap buffer for I/O operations.
---
.../org/apache/sis/storage/StorageConnector.java | 39 ++++++++++++----------
1 file changed, 21 insertions(+), 18 deletions(-)
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/storage/StorageConnector.java b/storage/sis-storage/src/main/java/org/apache/sis/storage/StorageConnector.java
index 0904b45..e04113f 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/storage/StorageConnector.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/storage/StorageConnector.java
@@ -86,7 +86,7 @@ import org.apache.sis.setup.OptionKey;
* is serializable.</p>
*
* @author Martin Desruisseaux (Geomatys)
- * @version 1.0
+ * @version 1.1
* @since 0.3
* @module
*/
@@ -99,13 +99,10 @@ public class StorageConnector implements Serializable {
/**
* The default size of the {@link ByteBuffer} to be created.
* Users can override this value by providing a value for {@link OptionKey#BYTE_BUFFER}.
- * Note that it usually don't need to be very large, since {@link RewindableLineReader}
- * will have its own buffer (which may be larger) and {@link ChannelDataInput} methods
- * writing in existing destination arrays will bypass the buffer.
*
* @see RewindableLineReader#BUFFER_SIZE
*/
- static final int DEFAULT_BUFFER_SIZE = 4096;
+ static final int DEFAULT_BUFFER_SIZE = 8192;
/**
* The minimal size of the {@link ByteBuffer} to be created. This size is used only
@@ -444,7 +441,7 @@ public class StorageConnector implements Serializable {
if ((cascade & CLEAR_ON_RESET) != 0) {
/*
* If the view can not be reset, in some cases we can discard it and recreate a new view when
- * first needed. The 'isValid' flag is left to false for telling that a new value is requested.
+ * first needed. The `isValid` flag is left to false for telling that a new value is requested.
*/
view = null;
return true;
@@ -467,7 +464,7 @@ public class StorageConnector implements Serializable {
/*
* ChannelDataInput can be recycled without the need to discard and recreate them. Note that
* this code requires that SeekableByteChannel has been seek to the channel beginning first.
- * This should be done by the above 'wrapperFor.reset()' call.
+ * This should be done by the above `wrapperFor.reset()` call.
*/
final ChannelDataInput input = (ChannelDataInput) view;
input.buffer.limit(0); // Must be after channel reset.
@@ -835,8 +832,8 @@ public class StorageConnector implements Serializable {
/*
* No instance has been created previously for the requested type. Open the stream now.
* Some types will need to reset the InputStream or Channel, but the decision of doing
- * so or not is left to openers. Result will be cached by the 'createFoo()' method.
- * Note that it may cache 'null' value if no stream of the given type can be created.
+ * so or not is left to openers. Result will be cached by the `createFoo()` method.
+ * Note that it may cache `null` value if no stream of the given type can be created.
*/
final Object view;
try {
@@ -933,14 +930,20 @@ public class StorageConnector implements Serializable {
}
/*
* ChannelDataInput depends on ReadableByteChannel, which itself depends on storage
- * (potentially an InputStream). We need to remember this chain in 'Coupled' objects.
+ * (potentially an InputStream). We need to remember this chain in `Coupled` objects.
*/
final String name = getStorageName();
final ReadableByteChannel channel = factory.readable(name, null);
addView(ReadableByteChannel.class, channel, null, factory.isCoupled() ? CASCADE_ON_RESET : 0);
ByteBuffer buffer = getOption(OptionKey.BYTE_BUFFER); // User-supplied buffer.
if (buffer == null) {
- buffer = ByteBuffer.allocate(DEFAULT_BUFFER_SIZE); // Default buffer if user did not specified any.
+ /*
+ * If the user did not specified a buffer, creates one now. We use a direct buffer for better
+ * leveraging of `ChannelDataInput`, which tries hard to transfer data in the most direct way
+ * between buffers and arrays. By contrast creating a heap buffer would have implied the use
+ * of a temporary direct buffer cached by the JDK itself (in JDK internal implementation).
+ */
+ buffer = ByteBuffer.allocateDirect(DEFAULT_BUFFER_SIZE);
}
final ChannelDataInput asDataInput;
if (asImageInputStream) {
@@ -1042,7 +1045,7 @@ public class StorageConnector implements Serializable {
final int n = in.read(buffer);
in.reset();
if (n >= 1) {
- // Can not invoke asReadOnly() because 'prefetch()' need to be able to write in it.
+ // Can not invoke asReadOnly() because `prefetch()` needs to be able to write in it.
asByteBuffer = ByteBuffer.wrap(buffer).order(in.getByteOrder());
asByteBuffer.limit(n);
}
@@ -1297,7 +1300,7 @@ public class StorageConnector implements Serializable {
Object v = c.view;
if (v != view) {
if (v instanceof AutoCloseable) {
- toClose.putIfAbsent((AutoCloseable) v, Boolean.TRUE); // Mark 'v' as needing to be closed.
+ toClose.putIfAbsent((AutoCloseable) v, Boolean.TRUE); // Mark `v` as needing to be closed.
}
} else {
/*
@@ -1309,7 +1312,7 @@ public class StorageConnector implements Serializable {
do {
v = c.view;
if (v instanceof AutoCloseable) {
- toClose.put((AutoCloseable) v, Boolean.FALSE); // Protect 'v' against closing.
+ toClose.put((AutoCloseable) v, Boolean.FALSE); // Protect `v` against closing.
}
c = c.wrapperFor;
} while (c != null);
@@ -1327,14 +1330,14 @@ public class StorageConnector implements Serializable {
* The "AutoCloseable.close() is not indempotent" problem
* ------------------------------------------------------
* We will need a set of objects to close without duplicated values. For example the values associated to the
- * 'ImageInputStream.class' and 'DataInput.class' keys are often the same instance. We must avoid duplicated
- * values because 'ImageInputStream.close()' is not indempotent, i.e. invoking their 'close()' method twice
- * will thrown an IOException.
+ * `ImageInputStream.class` and `DataInput.class` keys are often the same instance. We must avoid duplicated
+ * values because `ImageInputStream.close()` is not indempotent, i.e. invoking their `close()` method twice
+ * will throw an IOException.
*
* Generally speaking, all AutoCloseable instances are not guaranteed to be indempotent because this is not
* required by the interface contract. Consequently we must be careful to not invoke the close() method on
* the same instance twice (indirectly or indirectly). An exception to this rule is ImageInputStream, which
- * does not close its underlying stream. Those exceptions are identified by 'cascadeOnClose' set to 'true'.
+ * does not close its underlying stream. Those exceptions are identified by `cascadeOnClose` set to `true`.
*/
if (!toClose.isEmpty()) {
for (Coupled c : views.values()) {
[sis] 01/02: Expand the support to all primitive types.
Posted by de...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit 54eb82c04bc9fbbb830270d14e21c47643482a84
Author: Martin Desruisseaux <ma...@geomatys.com>
AuthorDate: Thu Jul 29 18:44:03 2021 +0200
Expand the support to all primitive types.
---
.../apache/sis/internal/geotiff/Uncompressed.java | 228 +++++++++++++++++++--
1 file changed, 214 insertions(+), 14 deletions(-)
diff --git a/storage/sis-geotiff/src/main/java/org/apache/sis/internal/geotiff/Uncompressed.java b/storage/sis-geotiff/src/main/java/org/apache/sis/internal/geotiff/Uncompressed.java
index 9b695a6..2486a90 100644
--- a/storage/sis-geotiff/src/main/java/org/apache/sis/internal/geotiff/Uncompressed.java
+++ b/storage/sis-geotiff/src/main/java/org/apache/sis/internal/geotiff/Uncompressed.java
@@ -19,6 +19,11 @@ package org.apache.sis.internal.geotiff;
import java.io.IOException;
import java.nio.Buffer;
import java.nio.ByteBuffer;
+import java.nio.ShortBuffer;
+import java.nio.IntBuffer;
+import java.nio.LongBuffer;
+import java.nio.FloatBuffer;
+import java.nio.DoubleBuffer;
import org.apache.sis.internal.storage.io.ChannelDataInput;
import org.apache.sis.util.resources.Errors;
import org.apache.sis.util.Classes;
@@ -78,7 +83,12 @@ public abstract class Uncompressed extends Inflater {
public static Uncompressed create(final ChannelDataInput input, final long start,
final int count, final int size, final int[] skips, final Buffer target)
{
- if (target instanceof ByteBuffer) return new Bytes(input, start, count, size, skips, (ByteBuffer) target);
+ if (target instanceof ByteBuffer) return new Bytes (input, start, count, size, skips, (ByteBuffer) target);
+ if (target instanceof ShortBuffer) return new Shorts (input, start, count, size, skips, (ShortBuffer) target);
+ if (target instanceof IntBuffer) return new Ints (input, start, count, size, skips, (IntBuffer) target);
+ if (target instanceof LongBuffer) return new Longs (input, start, count, size, skips, (LongBuffer) target);
+ if (target instanceof FloatBuffer) return new Floats (input, start, count, size, skips, (FloatBuffer) target);
+ if (target instanceof DoubleBuffer) return new Doubles(input, start, count, size, skips, (DoubleBuffer) target);
throw new IllegalArgumentException(Errors.format(Errors.Keys.UnsupportedType_1, Classes.getClass(target)));
}
@@ -114,6 +124,32 @@ public abstract class Uncompressed extends Inflater {
}
/**
+ * Skips the number of elements specified by the {@link #skipAfterChunks} array at the given index.
+ * This method tries to move by incrementing the buffer position.
+ *
+ * <div class="note"><b>Design note:</b>
+ * we do not use {@link ChannelDataInput#seek(long)} because the displacement is usually small.
+ * Changing the buffer position is sufficient in the majority of cases. If not, then it should
+ * be okay to fill the buffer with next data (instead of doing a seek operation) because there
+ * is usually few remaining values to skip. Performance of this method is important, so we try
+ * to avoid overhead.</div>
+ *
+ * @param skipIndex index in {@code skipAfterChunks} array.
+ * @return new {@code skipIndex} value.
+ */
+ final int skipAfterChunk(int skipIndex) throws IOException {
+ int n = skipAfterChunks[skipIndex] * sampleSize;
+ while (n != 0) {
+ final int p = input.buffer.position();
+ final int s = Math.min(p + n, input.buffer.limit());
+ input.buffer.position(s);
+ if ((n -= (s - p)) == 0) break;
+ input.ensureBufferContains(sampleSize);
+ }
+ return (++skipIndex < skipAfterChunks.length) ? skipIndex : 0;
+ }
+
+ /**
* Inflater for sample values stored as bytes.
*/
private static final class Bytes extends Uncompressed {
@@ -126,24 +162,17 @@ public abstract class Uncompressed extends Inflater {
this.target = target;
}
- /** Reads and decompress a row of sample values. */
+ /** Reads a row of sample values. */
@Override public void uncompressRow() throws IOException {
super.uncompressRow();
- int ip = 0;
+ int skipIndex = 0;
for (int i = chunksPerRow; --i > 0;) { // (chunksPerRow - 1) iterations.
int n = samplesPerChunk;
- do target.put(input.readByte());
+ input.ensureBufferContains(n);
+ do target.put(input.buffer.get()); // Number of iterations should be low (often 1).
while (--n != 0);
- /*
- * Following loop is executed only if there is subsampling on the X axis.
- * We invoke `readByte()` in a loop instead of invoking `skip` because if
- * the number of bytes to skip is small, this is more efficient.
- */
if (skipAfterChunks != null) {
- for (n = skipAfterChunks[ip]; --n >= 0;) {
- input.readByte();
- }
- if (++ip >= skipAfterChunks.length) ip = 0;
+ skipIndex = skipAfterChunk(skipIndex);
}
}
/*
@@ -153,7 +182,178 @@ public abstract class Uncompressed extends Inflater {
* is in the last column of the tile.
*/
int n = samplesPerChunk;
- do target.put(input.readByte());
+ input.ensureBufferContains(n);
+ do target.put(input.buffer.get());
+ while (--n != 0);
+ }
+ }
+
+ /**
+ * Inflater for sample values stored as short integers.
+ * This is a copy of {@link Bytes} implementation with only the type changed.
+ */
+ private static final class Shorts extends Uncompressed {
+ /** Where to copy the values that we will read. */
+ private final ShortBuffer target;
+
+ /** Creates a new inflater which will write in the given buffer. */
+ Shorts(ChannelDataInput input, long start, int count, int size, int[] skips, ShortBuffer target) {
+ super(input, start, count, size, skips, Short.BYTES);
+ this.target = target;
+ }
+
+ /** Reads a row of sample values. */
+ @Override public void uncompressRow() throws IOException {
+ super.uncompressRow();
+ int skipIndex = 0;
+ for (int i = chunksPerRow; --i > 0;) {
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Short.BYTES);
+ do target.put(input.buffer.getShort());
+ while (--n != 0);
+ if (skipAfterChunks != null) {
+ skipIndex = skipAfterChunk(skipIndex);
+ }
+ }
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Short.BYTES);
+ do target.put(input.buffer.getShort());
+ while (--n != 0);
+ }
+ }
+
+ /**
+ * Inflater for sample values stored as 32 bits integers.
+ * This is a copy of {@link Bytes} implementation with only the type changed.
+ */
+ private static final class Ints extends Uncompressed {
+ /** Where to copy the values that we will read. */
+ private final IntBuffer target;
+
+ /** Creates a new inflater which will write in the given buffer. */
+ Ints(ChannelDataInput input, long start, int count, int size, int[] skips, IntBuffer target) {
+ super(input, start, count, size, skips, Integer.BYTES);
+ this.target = target;
+ }
+
+ /** Reads a row of sample values. */
+ @Override public void uncompressRow() throws IOException {
+ super.uncompressRow();
+ int skipIndex = 0;
+ for (int i = chunksPerRow; --i > 0;) {
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Integer.BYTES);
+ do target.put(input.buffer.getInt());
+ while (--n != 0);
+ if (skipAfterChunks != null) {
+ skipIndex = skipAfterChunk(skipIndex);
+ }
+ }
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Integer.BYTES);
+ do target.put(input.buffer.getInt());
+ while (--n != 0);
+ }
+ }
+
+ /**
+ * Inflater for sample values stored as long integers.
+ * This is a copy of {@link Bytes} implementation with only the type changed.
+ */
+ private static final class Longs extends Uncompressed {
+ /** Where to copy the values that we will read. */
+ private final LongBuffer target;
+
+ /** Creates a new inflater which will write in the given buffer. */
+ Longs(ChannelDataInput input, long start, int count, int size, int[] skips, LongBuffer target) {
+ super(input, start, count, size, skips, Long.BYTES);
+ this.target = target;
+ }
+
+ /** Reads a row of sample values. */
+ @Override public void uncompressRow() throws IOException {
+ super.uncompressRow();
+ int skipIndex = 0;
+ for (int i = chunksPerRow; --i > 0;) {
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Long.BYTES);
+ do target.put(input.buffer.getLong());
+ while (--n != 0);
+ if (skipAfterChunks != null) {
+ skipIndex = skipAfterChunk(skipIndex);
+ }
+ }
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Long.BYTES);
+ do target.put(input.buffer.getLong());
+ while (--n != 0);
+ }
+ }
+
+ /**
+ * Inflater for sample values stored as single-precision floating point numbers.
+ * This is a copy of {@link Bytes} implementation with only the type changed.
+ */
+ private static final class Floats extends Uncompressed {
+ /** Where to copy the values that we will read. */
+ private final FloatBuffer target;
+
+ /** Creates a new inflater which will write in the given buffer. */
+ Floats(ChannelDataInput input, long start, int count, int size, int[] skips, FloatBuffer target) {
+ super(input, start, count, size, skips, Float.BYTES);
+ this.target = target;
+ }
+
+ /** Reads a row of sample values. */
+ @Override public void uncompressRow() throws IOException {
+ super.uncompressRow();
+ int skipIndex = 0;
+ for (int i = chunksPerRow; --i > 0;) {
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Float.BYTES);
+ do target.put(input.buffer.getFloat());
+ while (--n != 0);
+ if (skipAfterChunks != null) {
+ skipIndex = skipAfterChunk(skipIndex);
+ }
+ }
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Float.BYTES);
+ do target.put(input.buffer.getFloat());
+ while (--n != 0);
+ }
+ }
+
+ /**
+ * Inflater for sample values stored as double-precision floating point numbers.
+ * This is a copy of {@link Bytes} implementation with only the type changed.
+ */
+ private static final class Doubles extends Uncompressed {
+ /** Where to copy the values that we will read. */
+ private final DoubleBuffer target;
+
+ /** Creates a new inflater which will write in the given buffer. */
+ Doubles(ChannelDataInput input, long start, int count, int size, int[] skips, DoubleBuffer target) {
+ super(input, start, count, size, skips, Double.BYTES);
+ this.target = target;
+ }
+
+ /** Reads a row of sample values. */
+ @Override public void uncompressRow() throws IOException {
+ super.uncompressRow();
+ int skipIndex = 0;
+ for (int i = chunksPerRow; --i > 0;) {
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Double.BYTES);
+ do target.put(input.buffer.getDouble());
+ while (--n != 0);
+ if (skipAfterChunks != null) {
+ skipIndex = skipAfterChunk(skipIndex);
+ }
+ }
+ int n = samplesPerChunk;
+ input.ensureBufferContains(n * Double.BYTES);
+ do target.put(input.buffer.getDouble());
while (--n != 0);
}
}