You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sis.apache.org by de...@apache.org on 2013/05/23 23:26:20 UTC
svn commit: r1485856 - in /sis/branches/JDK7:
core/sis-utility/src/main/java/org/apache/sis/util/resources/
storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/
storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/
storage/s...
Author: desruisseaux
Date: Thu May 23 21:26:19 2013
New Revision: 1485856
URL: http://svn.apache.org/r1485856
Log:
Moved some NIO services out of ChannelDecoder, in a new DataInputChannel (by analogy with DataInputStream).
The intend is to make easier to reuse those services in VariableInfo for reading the actual variable data.
Added:
sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/
sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/DataInputChannel.java
- copied, changed from r1485646, sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/package-info.java (with props)
sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/
sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/DataInputChannelCompleted.java (with props)
Modified:
sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.java
sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.properties
sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors_fr.properties
sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/test/suite/NetcdfTestSuite.java
Modified: sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.java?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.java [UTF-8] (original)
+++ sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.java [UTF-8] Thu May 23 21:26:19 2013
@@ -431,6 +431,11 @@ public final class Errors extends Indexe
public static final int StalledThread_1 = 63;
/**
+ * Can not move backward in the “{0}” stream.
+ */
+ public static final int StreamIsForwardOnly_1 = 95;
+
+ /**
* Ordering between “{0}” and “{1}” elements is undefined.
*/
public static final int UndefinedOrderingForElements_2 = 70;
Modified: sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.properties
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.properties?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.properties [ISO-8859-1] (original)
+++ sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors.properties [ISO-8859-1] Thu May 23 21:26:19 2013
@@ -98,6 +98,7 @@ OddArrayLength_1 = Array
RecursiveCreateCallForKey_1 = Recursive call while creating an object for the \u201c{0}\u201d key.
RequireDecimalSeparator = A decimal separator is required.
StalledThread_1 = Thread \u201c{0}\u201d seems stalled.
+StreamIsForwardOnly_1 = Can not move backward in the \u201c{0}\u201d stream.
UndefinedOrderingForElements_2 = Ordering between \u201c{0}\u201d and \u201c{1}\u201d elements is undefined.
UnexpectedChange_1 = Unexpected change in \u2018{0}\u2019.
UnexpectedEndOfFile_1 = Unexpected end of file while reading \u201c{0}\u201d.
Modified: sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors_fr.properties
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors_fr.properties?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors_fr.properties [ISO-8859-1] (original)
+++ sis/branches/JDK7/core/sis-utility/src/main/java/org/apache/sis/util/resources/Errors_fr.properties [ISO-8859-1] Thu May 23 21:26:19 2013
@@ -87,6 +87,7 @@ OddArrayLength_1 = La lon
RecursiveCreateCallForKey_1 = Appel r\u00e9cursif lors de la cr\u00e9ation d\u2019un objet pour la cl\u00e9 \u201c{0}\u201d.
RequireDecimalSeparator = Un s\u00e9parateur d\u00e9cimal est requis.
StalledThread_1 = La t\u00e2che \u201c{0}\u201d semble bloqu\u00e9e.
+StreamIsForwardOnly_1 = Ne peut pas reculer dans le flux de donn\u00e9es \u201c{0}\u201d.
UndefinedOrderingForElements_2 = L\u2019ordre entre les \u00e9l\u00e9ments \u201c{0}\u201d et \u201c{1}\u201d n\u2019est pas d\u00e9fini.
UnexpectedChange_1 = Changement inattendu dans \u2018{0}\u2019.
UnexpectedEndOfFile_1 = Fin de fichier inattendue lors de la lecture de \u201c{0}\u201d.
Modified: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java [UTF-8] (original)
+++ sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java [UTF-8] Thu May 23 21:26:19 2013
@@ -16,6 +16,7 @@
*/
package org.apache.sis.internal.netcdf;
+import java.io.IOException;
import java.awt.image.DataBuffer;
import org.apache.sis.util.Classes;
import org.apache.sis.util.Debug;
@@ -194,6 +195,14 @@ public abstract class Variable {
public abstract Object[] getAttributeValues(String attributeName, boolean numeric);
/**
+ * Reads all the data for this variable and returns them as an array of a Java primitive type.
+ *
+ * @return The data as an array of a Java primitive type.
+ * @throws IOException If an error occurred while reading the data.
+ */
+ public abstract Object read() throws IOException;
+
+ /**
* Returns a string representation of this variable for debugging purpose.
*/
@Debug
Modified: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java [UTF-8] (original)
+++ sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java [UTF-8] Thu May 23 21:26:19 2013
@@ -27,7 +27,6 @@ import java.util.Date;
import java.util.Locale;
import java.util.regex.Pattern;
import java.io.IOException;
-import java.io.EOFException;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.lang.reflect.Array;
@@ -39,6 +38,7 @@ import org.apache.sis.internal.netcdf.De
import org.apache.sis.internal.netcdf.Variable;
import org.apache.sis.internal.netcdf.GridGeometry;
import org.apache.sis.internal.netcdf.WarningProducer;
+import org.apache.sis.internal.storage.DataInputChannel;
import org.apache.sis.internal.util.CollectionsExt;
import org.apache.sis.internal.jdk8.JDK8;
import org.apache.sis.storage.DataStoreException;
@@ -118,21 +118,9 @@ public final class ChannelDecoder extend
private static final int DIMENSION = 0x0A, VARIABLE = 0x0B, ATTRIBUTE = 0x0C;
/**
- * A file identifier used only for formatting error message.
+ * The {@link ReadableByteChannel} together with a {@link ByteBuffer} for reading the data.
*/
- private final String filename;
-
- /**
- * The channel from where data are read.
- * This is supplied at construction time.
- */
- private final ReadableByteChannel channel;
-
- /**
- * The buffer to use for transferring data from the channel to memory, mostly at header reading time.
- * Must be backed by a Java {@code byte[]} array, because we will occasionally reference that array.
- */
- private final ByteBuffer buffer;
+ private final DataInputChannel input;
/**
* {@code false} if the file is the classic format, or
@@ -195,16 +183,14 @@ public final class ChannelDecoder extend
throws IOException, DataStoreException
{
super(parent);
- this.filename = filename;
- this.channel = channel;
- this.buffer = ByteBuffer.allocate(1024);
- channel.read(buffer);
- buffer.flip();
+ // The buffer must be backed by a Java {@code byte[]} array,
+ // because we will occasionally reference that array.
+ input = new DataInputChannel(filename, channel, ByteBuffer.allocate(4096), false);
/*
* Check the magic number, which is expected to be exactly 3 bytes forming the "CDF" string.
* The 4th byte is the version number, which we opportunistically use after the magic number check.
*/
- int version = readInt();
+ int version = input.readInt();
if ((version & 0xFFFFFF00) != (('C' << 24) | ('D' << 16) | ('F' << 8))) {
throw new DataStoreException(Errors.format(Errors.Keys.UnexpectedFileFormat_2, "NetCDF", filename));
}
@@ -217,7 +203,7 @@ public final class ChannelDecoder extend
case 2: is64bits = true; break;
default: throw new DataStoreException(Errors.format(Errors.Keys.UnsupportedVersion_1, version));
}
- numrecs = readInt();
+ numrecs = input.readInt();
/*
* Read the dimension, attribute and variable declarations. We expect exactly 3 lists,
* where any of them can be flagged as absent by a long (64 bits) 0.
@@ -226,7 +212,7 @@ public final class ChannelDecoder extend
VariableInfo[] variables = null;
Attribute[] attributes = null;
for (int i=0; i<3; i++) {
- final long tn = readLong(); // Combination of tag and nelems
+ final long tn = input.readLong(); // Combination of tag and nelems
if (tn != 0) {
final int tag = (int) (tn >>> Integer.SIZE);
final int nelems = (int) tn;
@@ -266,7 +252,7 @@ public final class ChannelDecoder extend
* that the file should be a NetCDF one, but we found some inconsistency or unknown tags.
*/
private DataStoreException malformedHeader() {
- return new DataStoreException(Errors.format(Errors.Keys.CanNotParseFile_2, "NetCDF", filename));
+ return new DataStoreException(Errors.format(Errors.Keys.CanNotParseFile_2, "NetCDF", input.filename));
}
/**
@@ -275,7 +261,7 @@ public final class ChannelDecoder extend
private void ensureNonNegative(final int nelems, final int tag) throws DataStoreException {
if (nelems < 0) {
throw new DataStoreException(Errors.format(Errors.Keys.NegativeArrayLength_1,
- filename + DefaultNameSpace.DEFAULT_SEPARATOR + tagName(tag)));
+ input.filename + DefaultNameSpace.DEFAULT_SEPARATOR + tagName(tag)));
}
}
@@ -293,65 +279,24 @@ public final class ChannelDecoder extend
* @param name The name of the element to read, used only in case of error for formatting the message.
* @return The number of bytes to read, rounded to the next multiple of 4.
*/
- private int require(final int n, final int dataSize, String name) throws IOException, DataStoreException {
+ private int ensureBufferContains(final int n, final int dataSize, String name) throws IOException, DataStoreException {
// (n+3) & ~3 is a trick for rounding 'n' to the next multiple of 4.
final long size = ((n & 0xFFFFFFFFL) * dataSize + 3) & ~3;
- if (size > buffer.capacity()) {
- name = filename + DefaultNameSpace.DEFAULT_SEPARATOR + name;
+ if (size > input.buffer.capacity()) {
+ name = input.filename + DefaultNameSpace.DEFAULT_SEPARATOR + name;
throw new DataStoreException(n < 0 ?
Errors.format(Errors.Keys.NegativeArrayLength_1, name) :
Errors.format(Errors.Keys.ExcessiveListSize_2, name, n));
}
- require((int) size);
+ input.ensureBufferContains((int) size);
return (int) size;
}
/**
- * Makes sure that the buffer contains at least <var>n</var> remaining bytes.
- * It is caller's responsibility to ensure that the given number of bytes is
- * not greater than the buffer capacity.
- *
- * @param n The minimal number of bytes needed in the {@linkplain #buffer}.
- * @throws EOFException If the channel has reached the end of stream.
- * @throws IOException If an other kind of error occurred while reading.
- */
- private void require(int n) throws IOException {
- assert n <= buffer.capacity() : n;
- n -= buffer.remaining();
- if (n > 0) {
- buffer.compact();
- do {
- final int c = channel.read(buffer);
- if (c < 0) {
- throw new EOFException(Errors.format(Errors.Keys.UnexpectedEndOfFile_1, filename));
- }
- n -= c;
- } while (n > 0);
- buffer.flip();
- }
- }
-
- /**
- * Reads the next integer value from the buffer.
- */
- private int readInt() throws IOException {
- require(Integer.SIZE / Byte.SIZE);
- return buffer.getInt();
- }
-
- /**
- * Reads the next long value from the buffer.
- */
- private long readLong() throws IOException {
- require(Long.SIZE / Byte.SIZE);
- return buffer.getLong();
- }
-
- /**
* Reads the next offset, which may be encoded on 32 or 64 bits depending on the file format.
*/
private long readOffset() throws IOException {
- return is64bits ? readLong() : (readInt() & 0xFFFFFFFFL);
+ return is64bits ? input.readLong() : input.readUnsignedInt();
}
/**
@@ -359,11 +304,12 @@ public final class ChannelDecoder extend
* variable and attribute names in the header. Note that attribute value may have a different encoding.
*/
private String readName() throws IOException, DataStoreException {
- final int length = readInt();
+ final int length = input.readInt();
if (length < 0) {
throw malformedHeader();
}
- final int size = require(length, 1, "<name>");
+ final ByteBuffer buffer = input.buffer;
+ final int size = ensureBufferContains(length, 1, "<name>");
final int position = buffer.position(); // Must be after 'require'
final String text = new String(buffer.array(), position, length, NAME_ENCODING);
buffer.position(position + size);
@@ -381,7 +327,8 @@ public final class ChannelDecoder extend
if (length == 0) {
return null;
}
- final int size = require(length, VariableInfo.sizeOf(type), name);
+ final ByteBuffer buffer = input.buffer;
+ final int size = ensureBufferContains(length, VariableInfo.sizeOf(type), name);
final int position = buffer.position(); // Must be after 'require'
final Object result;
switch (type) {
@@ -443,7 +390,7 @@ public final class ChannelDecoder extend
final Dimension[] dimensions = new Dimension[nelems];
for (int i=0; i<nelems; i++) {
final String name = readName();
- int length = readInt();
+ int length = input.readInt();
if (length == 0) {
length = numrecs;
if (length == STREAMING) {
@@ -473,7 +420,7 @@ public final class ChannelDecoder extend
int count = 0;
for (int i=0; i<nelems; i++) {
final String name = readName();
- final Object value = readValues(name, readInt(), readInt());
+ final Object value = readValues(name, input.readInt(), input.readInt());
if (value != null) {
attributes[count++] = new Attribute(name, value);
}
@@ -510,11 +457,11 @@ public final class ChannelDecoder extend
final VariableInfo[] variables = new VariableInfo[nelems];
for (int j=0; j<nelems; j++) {
final String name = readName();
- final int n = readInt();
+ final int n = input.readInt();
final Dimension[] varDims = new Dimension[n];
try {
for (int i=0; i<n; i++) {
- varDims[i] = dimensions[readInt()];
+ varDims[i] = dimensions[input.readInt()];
}
} catch (IndexOutOfBoundsException cause) {
final DataStoreException e = malformedHeader();
@@ -526,7 +473,7 @@ public final class ChannelDecoder extend
* but with less cases in the "switch" statements.
*/
Attribute[] attributes = null;
- final long tn = readLong();
+ final long tn = input.readLong();
if (tn != 0) {
final int tag = (int) (tn >>> Integer.SIZE);
final int na = (int) tn;
@@ -537,8 +484,8 @@ public final class ChannelDecoder extend
default: throw malformedHeader();
}
}
- variables[j] = new VariableInfo(name, varDims, dimensions,
- toMap(attributes, Attribute.NAME_FUNCTION), readInt(), readInt(), readOffset());
+ variables[j] = new VariableInfo(input, name, varDims, dimensions,
+ toMap(attributes, Attribute.NAME_FUNCTION), input.readInt(), input.readInt(), readOffset());
}
return variables;
}
@@ -762,6 +709,6 @@ nextVar: for (final VariableInfo vari
*/
@Override
public void close() throws IOException {
- channel.close();
+ input.channel.close();
}
}
Modified: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java [UTF-8] (original)
+++ sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java [UTF-8] Thu May 23 21:26:19 2013
@@ -17,10 +17,12 @@
package org.apache.sis.internal.netcdf.impl;
import java.util.Map;
+import java.io.IOException;
import ucar.nc2.constants.CF;
import ucar.nc2.constants.CDM;
import ucar.nc2.constants._Coordinate;
import org.apache.sis.internal.netcdf.Variable;
+import org.apache.sis.internal.storage.DataInputChannel;
/**
@@ -79,6 +81,13 @@ final class VariableInfo extends Variabl
};
/**
+ * The channel together with a buffer for reading the variable data.
+ *
+ * @see #read()
+ */
+ private final DataInputChannel input;
+
+ /**
* The variable name.
*/
private final String name;
@@ -118,16 +127,21 @@ final class VariableInfo extends Variabl
/**
* Creates a new variable.
*/
- VariableInfo(final String name, final Dimension[] dimensions, final Dimension[] allDimensions,
+ VariableInfo(final DataInputChannel input, final String name,
+ final Dimension[] dimensions, final Dimension[] allDimensions,
final Map<String,Attribute> attributes, final int datatype, final int size, final long offset)
{
+ this.input = input;
this.name = name;
this.dimensions = dimensions;
this.allDimensions = allDimensions;
this.attributes = attributes;
this.datatype = datatype;
this.offset = offset;
- // TODO: verify 'size'.
+ /*
+ * The 'size' value is provided in the NetCDF files, but doesn't need to be stored since it
+ * is redundant with the dimension lengths and is not large enough for big variables anyway.
+ */
}
/**
@@ -251,4 +265,13 @@ final class VariableInfo extends Variabl
}
return new Object[0];
}
+
+ /**
+ * Reads all the data for this variable and returns them as an array of a Java primitive type.
+ */
+ @Override
+ public Object read() throws IOException {
+ input.seek(offset);
+ throw new UnsupportedOperationException(); // TODO
+ }
}
Modified: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java [UTF-8] (original)
+++ sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java [UTF-8] Thu May 23 21:26:19 2013
@@ -17,6 +17,8 @@
package org.apache.sis.internal.netcdf.ucar;
import java.util.List;
+import java.io.IOException;
+import ucar.ma2.Array;
import ucar.nc2.Attribute;
import ucar.nc2.Dimension;
import ucar.nc2.VariableIF;
@@ -162,4 +164,13 @@ final class VariableWrapper extends Vari
}
return new Object[0];
}
+
+ /**
+ * Reads all the data for this variable and returns them as an array of a Java primitive type.
+ */
+ @Override
+ public Object read() throws IOException {
+ final Array array = variable.read();
+ return array.get1DJavaArray(array.getElementType());
+ }
}
Copied: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/DataInputChannel.java (from r1485646, sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java)
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/DataInputChannel.java?p2=sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/DataInputChannel.java&p1=sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java&r1=1485646&r2=1485856&rev=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java [UTF-8] (original)
+++ sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/DataInputChannel.java [UTF-8] Thu May 23 21:26:19 2013
@@ -14,316 +14,159 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.sis.internal.netcdf.impl;
+package org.apache.sis.internal.storage;
-import java.util.Set;
-import java.util.Map;
-import java.util.LinkedHashSet;
-import java.util.LinkedHashMap;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Locale;
-import java.util.regex.Pattern;
import java.io.IOException;
import java.io.EOFException;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
-import java.lang.reflect.Array;
-import javax.measure.converter.UnitConverter;
-import javax.measure.converter.ConversionException;
-import org.opengis.parameter.InvalidParameterCardinalityException;
-import org.apache.sis.internal.jdk8.Function;
-import org.apache.sis.internal.netcdf.Decoder;
-import org.apache.sis.internal.netcdf.Variable;
-import org.apache.sis.internal.netcdf.GridGeometry;
-import org.apache.sis.internal.netcdf.WarningProducer;
-import org.apache.sis.internal.util.CollectionsExt;
-import org.apache.sis.internal.jdk8.JDK8;
-import org.apache.sis.storage.DataStoreException;
-import org.apache.sis.util.iso.DefaultNameSpace;
+import java.nio.channels.SeekableByteChannel;
import org.apache.sis.util.resources.Errors;
-import org.apache.sis.util.resources.Vocabulary;
-import org.apache.sis.util.ArraysExt;
-import org.apache.sis.measure.Units;
+import org.apache.sis.util.Debug;
/**
- * Provides NetCDF decoding services as a standalone library.
- * The javadoc in this class uses the "file" word for the source of data, but
- * this implementation actually works with arbitrary {@link ReadableByteChannel}.
+ * Provides convenience methods for working with a ({@link ReadableByteChannel}, {@link ByteBuffer}) pair.
+ * The channel and the buffer must be supplied by the caller. It is okay if they have already been used
+ * before {@code DataInputChannel} creation.
+ *
+ * {@note This class provides no <code>close()</code> method since it doesn't "own" the channel.}
+ *
+ * {@section Relationship with <code>DataInput</code>}
+ * This class API is compatibly with the {@link java.io.DataInput} interface, so subclasses can implement that
+ * interface if they wish. This class does not implement {@code DataInput} itself because it is not needed for
+ * SIS purposes, and because {@code DataInput} has undesirable methods ({@code readLine()} and {@code readUTF()}).
+ * However the {@code DataInputChannelCompleted} class in the test directory implements the {@code DataInput}
+ * interface, both for testing API compatibility and in case we choose to implement that interface after all
+ * in a future SIS version.
*
- * @author Johann Sorel (Geomatys)
* @author Martin Desruisseaux (Geomatys)
- * @since 0.3
+ * @since 0.3 (derived from geotk-3.07)
* @version 0.3
* @module
- *
- * @see <a href="http://portal.opengeospatial.org/files/?artifact_id=43734">NetCDF Classic and 64-bit Offset Format (1.0)</a>
*/
-public final class ChannelDecoder extends Decoder {
- /**
- * The encoding of dimension, variable and attribute names. This is fixed to {@value} by the
- * NetCDF specification. Note however that the encoding of attribute values may be different.
- *
- * @see #encoding
- * @see #readName()
- */
- private static final String NAME_ENCODING = "UTF-8";
-
- /**
- * The locale of dimension, variable and attribute names. This is used for the conversion to
- * lower-cases before case-insensitive searches.
- *
- * @see #findAttribute(String)
- */
- private static final Locale NAME_LOCALE = Locale.US;
-
- /**
- * The pattern to use for separating the component of a time unit.
- * An example of time unit is "<cite>days since 1970-01-01T00:00:00Z</cite>".
- *
- * @see #numberToDate(String, Number[])
- */
- private static final Pattern TIME_UNIT_PATTERN = Pattern.compile("(?i)\\bsince\\b");
-
- /**
- * {@code true} if the default timezone is UTC, or {@code false} if it shall be the
- * {@linkplain java.util.TimeZone#getDefault() system default}.
- */
- private static final boolean DEFAULT_TIMEZONE_IS_UTC = true;
-
- /*
- * NOTE: the names of the static constants below this point match the names used in the Backus-Naur Form (BNF)
- * definitions in the NetCDF Classic and 64-bit Offset Format (1.0) specification (link in class javdoc),
- * with NC_ prefix omitted. The types of those constants match the expected type in the file.
- */
-
- /**
- * A {@link #numrecs} value indicating indeterminate record count.
- * This value allows streaming data.
- *
- * @see #numrecs
- */
- private static final int STREAMING = -1;
-
- /**
- * Tag for lists of dimensions, variables or attributes, as 32 bits value to be followed
- * by a 32 bits integer given the number of elements in the list ({@code nelems}).
- * A 64-bits zero means that there is no list (identified as {@code ABSENT} in the BNF).
- *
- * @see #tagName(int)
- */
- private static final int DIMENSION = 0x0A, VARIABLE = 0x0B, ATTRIBUTE = 0x0C;
-
+public class DataInputChannel {
/**
* A file identifier used only for formatting error message.
*/
- private final String filename;
+ public final String filename;
/**
* The channel from where data are read.
* This is supplied at construction time.
*/
- private final ReadableByteChannel channel;
-
- /**
- * The buffer to use for transferring data from the channel to memory, mostly at header reading time.
- * Must be backed by a Java {@code byte[]} array, because we will occasionally reference that array.
- */
- private final ByteBuffer buffer;
-
- /**
- * {@code false} if the file is the classic format, or
- * {@code true} if it is the 64-bits offset format.
- */
- private final boolean is64bits;
-
- /**
- * Number of records as an unsigned integer, or {@value #STREAMING} if undetermined.
- */
- private final int numrecs;
+ public final ReadableByteChannel channel;
/**
- * The character encoding of attribute values. This encoding does <strong>not</strong> apply to
- * dimension, variable and attribute names, which are fixed to UTF-8 as of NetCDF specification.
- *
- * The specification said: "Although the characters used in netCDF names must be encoded as UTF-8,
- * character data may use other encodings. The variable attribute “_Encoding” is reserved for this
- * purpose in future implementations."
- *
- * @todo Fixed to ISO-LATIN-1 for now, needs to be determined in a better way.
- *
- * @see #NAME_ENCODING
- * @see #readString(String)
- */
- private final String encoding = "ISO-8859-1";
-
- /**
- * The variables found in the NetCDF file.
- *
- * @see #getVariables()
+ * The buffer to use for transferring data from the channel to memory.
*/
- private final VariableInfo[] variables;
+ public final ByteBuffer buffer;
/**
- * The attributes found in the NetCDF file.
- *
- * @see #findAttribute(String)
+ * The position of the channel when this {@code DataInputChannel} has been created.
+ * This is almost always 0, but we allow other values in case the data to read are
+ * encompassed inside a bigger file.
*/
- private final Map<String,Attribute> attributeMap;
+ private final long channelOffset;
/**
- * The grid geometries, created when first needed.
- *
- * @see #getGridGeometries()
+ * The position in {@link #channel} where is located the {@link #buffer} value at index 0.
+ * This is initially zero and shall be updated every time {@link ByteBuffer#compact()} or
+ * {@link ByteBuffer#clear()} is invoked.
*/
- private transient GridGeometry[] gridGeometries;
+ private long bufferOffset;
/**
- * Creates a new decoder for the given file.
- * This constructor parses immediately the header.
+ * Creates a new input source for the given channel and using the given buffer.
+ * If the buffer already contains some data, then the {@code filled} argument shall be {@code true}.
+ * Otherwise (e.g. if it is a newly created buffer), then {@code filled} shall be {@code false}.
*
- * @param parent Where to send the warnings, or {@code null} if none.
* @param filename A file identifier used only for formatting error message.
* @param channel The channel from where data are read.
+ * @param buffer The buffer where to copy the data.
+ * @param filled {@code true} if the buffer already contains data, or {@code false} if it needs
+ * to be initially filled with some content read from the channel.
* @throws IOException If an error occurred while reading the channel.
- * @throws DataStoreException If the content of the given channel is not a NetCDF file.
*/
- public ChannelDecoder(final WarningProducer parent, final String filename, final ReadableByteChannel channel)
- throws IOException, DataStoreException
+ public DataInputChannel(final String filename, final ReadableByteChannel channel, final ByteBuffer buffer,
+ final boolean filled) throws IOException
{
- super(parent);
- this.filename = filename;
- this.channel = channel;
- this.buffer = ByteBuffer.allocate(1024);
- channel.read(buffer);
- buffer.flip();
- /*
- * Check the magic number, which is expected to be exactly 3 bytes forming the "CDF" string.
- * The 4th byte is the version number, which we opportunistically use after the magic number check.
- */
- int version = readInt();
- if ((version & 0xFFFFFF00) != (('C' << 24) | ('D' << 16) | ('F' << 8))) {
- throw new DataStoreException(Errors.format(Errors.Keys.UnexpectedFileFormat_2, "NetCDF", filename));
- }
- /*
- * Check the version number.
- */
- version &= 0xFF;
- switch (version) {
- case 1: is64bits = false; break;
- case 2: is64bits = true; break;
- default: throw new DataStoreException(Errors.format(Errors.Keys.UnsupportedVersion_1, version));
- }
- numrecs = readInt();
- /*
- * Read the dimension, attribute and variable declarations. We expect exactly 3 lists,
- * where any of them can be flagged as absent by a long (64 bits) 0.
- */
- Dimension[] dimensions = null;
- VariableInfo[] variables = null;
- Attribute[] attributes = null;
- for (int i=0; i<3; i++) {
- final long tn = readLong(); // Combination of tag and nelems
- if (tn != 0) {
- final int tag = (int) (tn >>> Integer.SIZE);
- final int nelems = (int) tn;
- ensureNonNegative(nelems, tag);
- switch (tag) {
- case DIMENSION: dimensions = readDimensions(nelems); break;
- case VARIABLE: variables = readVariables (nelems, dimensions); break;
- case ATTRIBUTE: attributes = readAttributes(nelems); break;
- default: throw malformedHeader();
- }
- }
+ this.filename = filename;
+ this.channel = channel;
+ this.buffer = buffer;
+ this.channelOffset = (channel instanceof SeekableByteChannel) ? ((SeekableByteChannel) channel).position() : 0;
+ if (!filled) {
+ channel.read(buffer);
+ buffer.flip();
}
- attributeMap = toMap(attributes, Attribute.NAME_FUNCTION);
- this.variables = variables;
}
/**
- * Return the localized name of the given tag.
+ * Invoked when a call to {@link ReadableByteChannel#read(ByteBuffer)} has returned zero.
+ * Note that this is unrelated to end-of-file, in which case {@code read} returns -1.
+ * A return value of 0 happen for example if the channel is a socket in non-blocking mode
+ * and the socket buffer has not yet received new data.
+ *
+ * <p>The current implementation sleeps an arbitrary amount of time before to allow a new try.
+ * We do that in order to avoid high CPU consumption when data are expected to take more than
+ * a few nanoseconds to arrive.</p>
*
- * @param tag One of {@link #DIMENSION}, {@link #VARIABLE} or {@link #ATTRIBUTE} constants.
- * @return The localized name of the given tag, or its hexadecimal number if the given value is not
- * one of the expected constants.
+ * @throws IOException If the implementation chooses to stop the reading process.
*/
- private static String tagName(final int tag) {
- final int key;
- switch (tag) {
- case DIMENSION: key = Vocabulary.Keys.Dimensions; break;
- case VARIABLE: key = Vocabulary.Keys.Variables; break;
- case ATTRIBUTE: key = Vocabulary.Keys.Attributes; break;
- default: return Integer.toHexString(tag);
+ protected void onEmptyChannelBuffer() throws IOException {
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ // Someone doesn't want to let us sleep. Go back to work.
}
- return Vocabulary.format(key);
}
/**
- * Returns an exception for a malformed header. This is used only after we have determined
- * that the file should be a NetCDF one, but we found some inconsistency or unknown tags.
- */
- private DataStoreException malformedHeader() {
- return new DataStoreException(Errors.format(Errors.Keys.CanNotParseFile_2, "NetCDF", filename));
- }
-
- /**
- * Ensures that {@code nelems} is not a negative value.
+ * Returns {@code true} if the buffer or the channel has at least one byte remaining.
+ * If the {@linkplain #buffer} has no remaining bytes, then this method will attempts
+ * to read at least one byte from the {@linkplain #channel}. If no bytes can be read
+ * because the channel has reached the end of stream, then this method returns {@code false}.
+ *
+ * @return {@code true} if the buffer contains at least one remaining byte.
+ * @throws IOException If it was necessary to read from the channel and this operation failed.
*/
- private void ensureNonNegative(final int nelems, final int tag) throws DataStoreException {
- if (nelems < 0) {
- throw new DataStoreException(Errors.format(Errors.Keys.NegativeArrayLength_1,
- filename + DefaultNameSpace.DEFAULT_SEPARATOR + tagName(tag)));
+ public final boolean hasRemaining() throws IOException {
+ if (buffer.hasRemaining()) {
+ return true;
}
- }
-
- /**
- * Makes sure that the buffer contains at least <var>n</var> remaining elements of the given size.
- * If the buffer does not have enough bytes available, more bytes will be read from the channel.
- * If the buffer capacity is not sufficient for reading the given amount of data, an exception
- * is thrown.
- *
- * <p>The NetCDF format add padding after bytes, characters and short integers in order to align
- * the data on multiple of 4 bytes. This method adds such padding to the number of bytes to read.</p>
- *
- * @param n The number of elements to read.
- * @param dataSize The size of each element, in bytes.
- * @param name The name of the element to read, used only in case of error for formatting the message.
- * @return The number of bytes to read, rounded to the next multiple of 4.
- */
- private int require(final int n, final int dataSize, String name) throws IOException, DataStoreException {
- // (n+3) & ~3 is a trick for rounding 'n' to the next multiple of 4.
- final long size = ((n & 0xFFFFFFFFL) * dataSize + 3) & ~3;
- if (size > buffer.capacity()) {
- name = filename + DefaultNameSpace.DEFAULT_SEPARATOR + name;
- throw new DataStoreException(n < 0 ?
- Errors.format(Errors.Keys.NegativeArrayLength_1, name) :
- Errors.format(Errors.Keys.ExcessiveListSize_2, name, n));
+ bufferOffset += buffer.position();
+ buffer.clear();
+ int c = channel.read(buffer);
+ while (c == 0) {
+ onEmptyChannelBuffer();
+ c = channel.read(buffer);
}
- require((int) size);
- return (int) size;
+ buffer.flip();
+ return c >= 0;
}
/**
* Makes sure that the buffer contains at least <var>n</var> remaining bytes.
* It is caller's responsibility to ensure that the given number of bytes is
- * not greater than the buffer capacity.
+ * not greater than the {@linkplain ByteBuffer#capacity() buffer capacity}.
*
* @param n The minimal number of bytes needed in the {@linkplain #buffer}.
* @throws EOFException If the channel has reached the end of stream.
* @throws IOException If an other kind of error occurred while reading.
*/
- private void require(int n) throws IOException {
+ public final void ensureBufferContains(int n) throws IOException {
assert n <= buffer.capacity() : n;
n -= buffer.remaining();
if (n > 0) {
+ bufferOffset += buffer.position();
buffer.compact();
do {
final int c = channel.read(buffer);
- if (c < 0) {
- throw new EOFException(Errors.format(Errors.Keys.UnexpectedEndOfFile_1, filename));
+ if (c <= 0) {
+ if (c != 0) {
+ throw new EOFException(eof());
+ }
+ onEmptyChannelBuffer();
}
n -= c;
} while (n > 0);
@@ -332,436 +175,247 @@ public final class ChannelDecoder extend
}
/**
- * Reads the next integer value from the buffer.
- */
- private int readInt() throws IOException {
- require(Integer.SIZE / Byte.SIZE);
- return buffer.getInt();
- }
-
- /**
- * Reads the next long value from the buffer.
- */
- private long readLong() throws IOException {
- require(Long.SIZE / Byte.SIZE);
- return buffer.getLong();
- }
-
- /**
- * Reads the next offset, which may be encoded on 32 or 64 bits depending on the file format.
+ * Makes sure that the buffer contains at least one remaining byte.
+ *
+ * @throws EOFException If the channel has reached the end of stream.
+ * @throws IOException If an other kind of error occurred while reading.
*/
- private long readOffset() throws IOException {
- return is64bits ? readLong() : (readInt() & 0xFFFFFFFFL);
+ private void ensureNonEmpty() throws IOException {
+ if (!hasRemaining()) {
+ throw new EOFException(eof());
+ }
}
/**
- * Reads a string from the buffer in the {@value #NAME_ENCODING}. This is suitable for the dimension,
- * variable and attribute names in the header. Note that attribute value may have a different encoding.
+ * Returns the "end of file" error message, for {@link EOFException} creations.
*/
- private String readName() throws IOException, DataStoreException {
- final int length = readInt();
- if (length < 0) {
- throw malformedHeader();
- }
- final int size = require(length, 1, "<name>");
- final int position = buffer.position(); // Must be after 'require'
- final String text = new String(buffer.array(), position, length, NAME_ENCODING);
- buffer.position(position + size);
- return text;
+ private String eof() {
+ return Errors.format(Errors.Keys.UnexpectedEndOfFile_1, filename);
}
/**
- * Returns the values of the given type. In the given type is {@code CHAR}, then this method returns the values
- * as a {@link String}. Otherwise this method returns the value as an array of the corresponding primitive type
- * and the given length.
+ * Reads the next byte value (8 bits) from the stream. This method ensures that there is at
+ * least 1 byte remaining in the buffer, reading new bytes from the channel if necessary,
+ * then delegates to {@link ByteBuffer#get()}.
*
- * @return The value, or {@code null} if it was an empty string or an empty array.
+ * @return The value of the next byte from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- private Object readValues(final String name, final int type, final int length) throws IOException, DataStoreException {
- if (length == 0) {
- return null;
- }
- final int size = require(length, VariableInfo.sizeOf(type), name);
- final int position = buffer.position(); // Must be after 'require'
- final Object result;
- switch (type) {
- case VariableInfo.CHAR: {
- final String text = new String(buffer.array(), position, length, encoding).trim();
- result = text.isEmpty() ? null : text;
- break;
- }
- case VariableInfo.BYTE: {
- final byte[] array = new byte[length];
- buffer.get(array);
- result = array;
- break;
- }
- case VariableInfo.SHORT: {
- final short[] array = new short[length];
- buffer.asShortBuffer().get(array);
- result = array;
- break;
- }
- case VariableInfo.INT: {
- final int[] array = new int[length];
- buffer.asIntBuffer().get(array);
- result = array;
- break;
- }
- case VariableInfo.FLOAT: {
- final float[] array = new float[length];
- buffer.asFloatBuffer().get(array);
- result = array;
- break;
- }
- case VariableInfo.DOUBLE: {
- final double[] array = new double[length];
- buffer.asDoubleBuffer().get(array);
- result = array;
- break;
- }
- default: {
- throw malformedHeader();
- }
- }
- buffer.position(position + size);
- return result;
+ public final byte readByte() throws IOException {
+ ensureBufferContains(Byte.SIZE / Byte.SIZE);
+ return buffer.get();
}
/**
- * Reads dimensions from the NetCDF file header. The record structure is:
+ * Reads the next unsigned byte value (8 bits) from the stream.
+ * The implementation is as below:
*
- * <ul>
- * <li>The dimension name (use {@link #readName()})</li>
- * <li>The dimension length (use {@link #readInt()})</li>
- * </ul>
- *
- * @param nelems The number of dimensions to read.
- * @return The dimensions in the order they are declared in the NetCDF file.
- */
- private Dimension[] readDimensions(final int nelems) throws IOException, DataStoreException {
- final Dimension[] dimensions = new Dimension[nelems];
- for (int i=0; i<nelems; i++) {
- final String name = readName();
- int length = readInt();
- if (length == 0) {
- length = numrecs;
- if (length == STREAMING) {
- throw new DataStoreException(Errors.format(Errors.Keys.MissingValueForProperty_1, "numrecs"));
- }
- }
- dimensions[i] = new Dimension(name, length);
- }
- return dimensions;
- }
-
- /**
- * Reads attribute values from the NetCDF file header. Current implementation has no restriction on
- * the location in the header where the NetCDF attribute can be declared. The record structure is:
+ * {@preformat java
+ * return readByte() & 0xFF;
+ * }
*
- * <ul>
- * <li>The attribute name (use {@link #readName()})</li>
- * <li>The attribute type (BYTE, SHORT, …) (use {@link #readInt()})</li>
- * <li>The number of values of the above type (use {@link #readInt()})</li>
- * <li>The actual values as a variable length list (use {@link #readValues(String,int,int)})</li>
- * </ul>
- *
- * @param nelems The number of attributes to read.
- */
- private Attribute[] readAttributes(final int nelems) throws IOException, DataStoreException {
- final Attribute[] attributes = new Attribute[nelems];
- int count = 0;
- for (int i=0; i<nelems; i++) {
- final String name = readName();
- final Object value = readValues(name, readInt(), readInt());
- if (value != null) {
- attributes[count++] = new Attribute(name, value);
- }
- }
- return ArraysExt.resize(attributes, count);
+ * @return The value of the next unsigned byte from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
+ */
+ public final int readUnsignedByte() throws IOException {
+ return readByte() & 0xFF;
}
/**
- * Reads information (not data) about all variables from the NetCDF file header.
- * The current implementation requires the dimensions to be read before the variables.
- * The record structure is:
- *
- * <ul>
- * <li>The variable name (use {@link #readName()})</li>
- * <li>The number of dimensions (use {@link #readInt()})</li>
- * <li>Index of all dimensions (use {@link #readInt()} <var>n</var> time)</li>
- * <li>The {@link #ATTRIBUTE} tag (use {@link #readInt()} - actually combined as a long with next item)</li>
- * <li>Number of attributes (use {@link #readInt()} - actually combined as a long with above item)</li>
- * <li>The attribute values (use {@link #readAttributes(int)})</li>
- * <li>The data type (BYTE, …) (use {@link #readInt()})</li>
- * <li>The variable size (use {@link #readInt()})</li>
- * <li>Offset where data begins (use {@link #readOffset()})</li>
- * </ul>
+ * Reads the next short value (16 bits) from the stream. This method ensures that there is at
+ * least 2 bytes remaining in the buffer, reading new bytes from the channel if necessary,
+ * then delegates to {@link ByteBuffer#getShort()}.
*
- * @param nelems The number of variables to read.
- * @param dimensions The dimensions previously read by {@link #readDimensions(int)}.
+ * @return The value of the next short from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- private VariableInfo[] readVariables(final int nelems, final Dimension[] dimensions)
- throws IOException, DataStoreException
- {
- if (dimensions == null) {
- throw malformedHeader();
- }
- final VariableInfo[] variables = new VariableInfo[nelems];
- for (int j=0; j<nelems; j++) {
- final String name = readName();
- final int n = readInt();
- final Dimension[] varDims = new Dimension[n];
- try {
- for (int i=0; i<n; i++) {
- varDims[i] = dimensions[readInt()];
- }
- } catch (IndexOutOfBoundsException cause) {
- final DataStoreException e = malformedHeader();
- e.initCause(cause);
- throw e;
- }
- /*
- * Following block is almost a copy-and-paste of similar block in the contructor,
- * but with less cases in the "switch" statements.
- */
- Attribute[] attributes = null;
- final long tn = readLong();
- if (tn != 0) {
- final int tag = (int) (tn >>> Integer.SIZE);
- final int na = (int) tn;
- ensureNonNegative(na, tag);
- switch (tag) {
- // More cases may be added later if it appears to exist.
- case ATTRIBUTE: attributes = readAttributes(na); break;
- default: throw malformedHeader();
- }
- }
- variables[j] = new VariableInfo(name, varDims, dimensions,
- toMap(attributes, Attribute.NAME_FUNCTION), readInt(), readInt(), readOffset());
- }
- return variables;
+ public final short readShort() throws IOException {
+ ensureBufferContains(Short.SIZE / Byte.SIZE);
+ return buffer.getShort();
}
/**
- * Creates a (<cite>name</cite>, <cite>element</cite>) mapping for the given array of elements.
- * If the name of an element is not all lower cases, then this method also adds an entry for the
- * lower cases version of that name in order to allow case-insensitive searches.
- *
- * <p>Code searching in the returned map shall ask for the original (non lower-case) name
- * <strong>before</strong> to ask for the lower-cases version of that name.</p>
+ * Reads the next unsigned short value (16 bits) from the stream.
+ * The implementation is as below:
*
- * @param <E> The type of elements.
- * @param elements The elements to store in the map, or {@code null} if none.
- * @param nameFunction The function for computing a name from an element.
- * @return A (<cite>name</cite>, <cite>element</cite>) mapping with lower cases entries where possible.
- * @throws DataStoreException If the same name is used for more than one element.
+ * {@preformat java
+ * return readShort() & 0xFFFF;
+ * }
*
- * @see #findAttribute(String)
+ * @return The value of the next unsigned short from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- private static <E> Map<String,E> toMap(final E[] elements, final Function<E,String> nameFunction) throws DataStoreException {
- try {
- return CollectionsExt.toCaseInsensitiveNameMap(Arrays.asList(elements), nameFunction, NAME_LOCALE);
- } catch (InvalidParameterCardinalityException e) {
- throw new DataStoreException(Errors.format(Errors.Keys.ValueAlreadyDefined_1, e.getParameterName()));
- }
+ public final int readUnsignedShort() throws IOException {
+ return readShort() & 0xFFFF;
}
-
-
- // --------------------------------------------------------------------------------------------
- // Decoder API begins below this point. Above code was specific to parsing of NetCDF header.
- // --------------------------------------------------------------------------------------------
-
/**
- * Defines the groups where to search for named attributes, in preference order.
- * The {@code null} group name stands for the global attributes.
+ * Reads the next character (16 bits) from the stream. This method ensures that there is at
+ * least 2 bytes remaining in the buffer, reading new bytes from the channel if necessary,
+ * then delegates to {@link ByteBuffer#getChar()}.
+ *
+ * @return The value of the next character from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- @Override
- public void setSearchPath(final String... groupNames) throws IOException {
- if (groupNames.length != 1 || groupNames[0] != null) {
- throw new UnsupportedOperationException();
- }
+ public final char readChar() throws IOException {
+ ensureBufferContains(Character.SIZE / Byte.SIZE);
+ return buffer.getChar();
}
/**
- * Returns the path which is currently set. The array returned by this method may be only
- * a subset of the array given to {@link #setSearchPath(String[])} since only the name of
- * groups which have been found in the NetCDF file are returned by this method.
+ * Reads the next integer value (32 bits) from the stream. This method ensures that there is at
+ * least 4 bytes remaining in the buffer, reading new bytes from the channel if necessary, then
+ * delegates to {@link ByteBuffer#getInt()}.
+ *
+ * @return The value of the next integer from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- @Override
- public String[] getSearchPath() throws IOException {
- return new String[1];
+ public final int readInt() throws IOException {
+ ensureBufferContains(Integer.SIZE / Byte.SIZE);
+ return buffer.getInt();
}
/**
- * Returns the NetCDF attribute of the given name, or {@code null} if none.
- * The {@code name} argument is typically (but is not restricted too) one of
- * the constants defined in the {@link AttributeNames} class.
+ * Reads the next unsigned integer value (32 bits) from the stream.
+ * The implementation is as below:
+ *
+ * {@preformat java
+ * return readInt() & 0xFFFFFFFFL;
+ * }
*
- * @param name The name of the attribute to search, or {@code null}.
- * @return The attribute, or {@code null} if none.
+ * @return The value of the next unsigned integer from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- private Attribute findAttribute(final String name) {
- Attribute attribute = attributeMap.get(name);
- if (attribute == null) {
- final String lower = name.toLowerCase(NAME_LOCALE);
- if (lower != name) { // Identity comparison is ok since this check is only an optimization for a common case.
- attribute = attributeMap.get(lower);
- }
- }
- return attribute;
+ public final long readUnsignedInt() throws IOException {
+ return readInt() & 0xFFFFFFFFL;
}
/**
- * Returns the value for the attribute of the given name, or {@code null} if none.
+ * Reads the next long value (64 bits) from the stream. This method ensures that there is at
+ * least 8 bytes remaining in the buffer, reading new bytes from the channel if necessary,
+ * then delegates to {@link ByteBuffer#getLong()}.
*
- * @param name The name of the attribute to search, or {@code null}.
- * @return The attribute value, or {@code null} if none or empty or if the given name was null.
+ * @return The value of the next integer from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- @Override
- public String stringValue(final String name) throws IOException {
- final Attribute attribute = findAttribute(name);
- if (attribute != null) {
- return attribute.value.toString();
- }
- return null;
+ public final long readLong() throws IOException {
+ ensureBufferContains(Long.SIZE / Byte.SIZE);
+ return buffer.getLong();
}
/**
- * Returns the value of the attribute of the given name as a number, or {@code null} if none.
- * If there is more than one numeric value, only the first one is returned.
+ * Reads the next float value (32 bits) from the stream. This method ensures that there is at
+ * least 4 bytes remaining in the buffer, reading new bytes from the channel if necessary,
+ * then delegates to {@link ByteBuffer#getFloat()}.
+ *
+ * @return The value of the next float from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- @Override
- public Number numericValue(final String name) throws IOException {
- final Attribute attribute = findAttribute(name);
- if (attribute != null && attribute.value != null) {
- if (attribute.value instanceof String) {
- return parseNumber((String) attribute.value);
- }
- return (Number) Array.get(attribute.value, 0);
- }
- return null;
+ public final float readFloat() throws IOException {
+ ensureBufferContains(Float.SIZE / Byte.SIZE);
+ return buffer.getFloat();
}
/**
- * Returns the value of the attribute of the given name as a date, or {@code null} if none.
- * If there is more than one numeric value, only the first one is returned.
+ * Reads the next double value (64 bits) from the stream. This method ensures that there is at
+ * least 8 bytes remaining in the buffer, reading new bytes from the channel if necessary,
+ * then delegates to {@link ByteBuffer#getDouble()}.
+ *
+ * @return The value of the next double from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- @Override
- public Date dateValue(final String name) throws IOException {
- final Attribute attribute = findAttribute(name);
- if (attribute != null) {
- if (attribute.value instanceof String) try {
- return JDK8.parseDateTime((String) attribute.value, DEFAULT_TIMEZONE_IS_UTC);
- } catch (IllegalArgumentException e) {
- warning("dateValue", e);
- }
- }
- return null;
+ public final double readDouble() throws IOException {
+ ensureBufferContains(Double.SIZE / Byte.SIZE);
+ return buffer.getDouble();
}
/**
- * Converts the given numerical values to date, using the information provided in the given unit symbol.
- * The unit symbol is typically a string like "<cite>days since 1970-01-01T00:00:00Z</cite>".
+ * Reads {@code dest.length} bytes from the stream, and stores them into
+ * {@code dest} starting at index 0. The implementation is as below:
+ *
+ * {@preformat java
+ * return readFully(dest, 0, dest.length);
+ * }
*
- * @param values The values to convert. May contains {@code null} elements.
- * @return The converted values. May contains {@code null} elements.
+ * @param dest An array of bytes to be written to.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- @Override
- public Date[] numberToDate(final String symbol, final Number... values) throws IOException {
- final Date[] dates = new Date[values.length];
- final String[] parts = TIME_UNIT_PATTERN.split(symbol);
- if (parts.length == 2) try {
- final UnitConverter converter = Units.valueOf(parts[0]).getConverterToAny(Units.MILLISECOND);
- final long epoch = JDK8.parseDateTime(parts[1], DEFAULT_TIMEZONE_IS_UTC).getTime();
- for (int i=0; i<values.length; i++) {
- final Number value = values[i];
- if (value != null) {
- dates[i] = new Date(epoch + Math.round(converter.convert(value.doubleValue())));
- }
- }
- } catch (ConversionException | IllegalArgumentException e) {
- warning("numberToDate", e);
- }
- return dates;
+ public final void readFully(final byte[] dest) throws IOException {
+ readFully(dest, 0, dest.length);
}
/**
- * Returns all variables found in the NetCDF file.
- * This method returns a direct reference to an internal array - do not modify.
+ * Reads {@code length} bytes from the stream, and stores them into
+ * {@code dest} starting at index {@code offset}.
+ *
+ * @param dest An array of bytes to be written to.
+ * @param offset The starting position within {@code dest} to write.
+ * @param length The number of bytes to read.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
*/
- @Override
- public Variable[] getVariables() throws IOException {
- return variables;
+ public final void readFully(final byte[] dest, int offset, int length) throws IOException {
+ while (length != 0) {
+ ensureNonEmpty();
+ final int n = Math.min(buffer.remaining(), length);
+ buffer.get(dest, offset, n);
+ offset += n;
+ length -= n;
+ }
}
/**
- * Returns all grid geometries found in the NetCDF file.
- * This method returns a direct reference to an internal array - do not modify.
+ * Moves to the given position in the stream, relative to the stream position at construction time.
+ *
+ * @param position The position where to move.
+ * @throws IOException If the stream can not be moved to the given position.
*/
- @Override
- public GridGeometry[] getGridGeometries() throws IOException {
- if (gridGeometries == null) {
+ public final void seek(final long position) throws IOException {
+ long p = position - bufferOffset;
+ if (p >= 0 && p <= buffer.limit()) {
/*
- * First, find all variables which are used as coordinate system axis. The keys are the
- * grid dimensions which are the domain of the variable (i.e. the sources of the conversion
- * from grid coordinates to CRS coordinates).
+ * Requested position is inside the current limits of the buffer.
*/
- final Map<Dimension, List<VariableInfo>> dimToAxes = new IdentityHashMap<>();
- for (final VariableInfo variable : variables) {
- if (variable.isCoordinateSystemAxis()) {
- for (final Dimension dimension : variable.dimensions) {
- CollectionsExt.addToMultiValuesMap(dimToAxes, dimension, variable);
- }
- }
- }
+ buffer.position((int) p);
+ } else if (channel instanceof SeekableByteChannel) {
/*
- * For each variables, gets the list of all axes associated to their dimensions. The association
- * is given by the above 'dimToVars' map. More than one variable may have the same dimensions,
- * and consequently the same axes, so we will remember the previously created instances in order
- * to share them.
+ * Requested position is outside the current limits of the buffer,
+ * but we can set the new position directly in the channel.
*/
- final Set<VariableInfo> axes = new LinkedHashSet<>(4);
- final Map<List<Dimension>, GridGeometryInfo> dimsToGG = new LinkedHashMap<>();
-nextVar: for (final VariableInfo variable : variables) {
- if (variable.isCoordinateSystemAxis()) {
- continue;
- }
- final List<Dimension> dimensions = Arrays.asList(variable.dimensions);
- GridGeometryInfo gridGeometry = dimsToGG.get(dimensions);
- if (gridGeometry == null) {
- /*
- * Found a new list of dimensions for which no axes have been created yet.
- * If and only if we can find all axes, then create the GridGeometryInfo.
- * This is a "all or nothing" operation.
- */
- for (final Dimension dimension : variable.dimensions) {
- final List<VariableInfo> axis = dimToAxes.get(dimension);
- if (axis == null) {
- axes.clear();
- continue nextVar;
- }
- axes.addAll(axis);
- }
- gridGeometry = new GridGeometryInfo(variable.dimensions, axes.toArray(new VariableInfo[axes.size()]));
- dimsToGG.put(dimensions, gridGeometry);
- axes.clear();
+ ((SeekableByteChannel) channel).position(channelOffset + position);
+ bufferOffset = position;
+ buffer.clear();
+ channel.read(buffer);
+ buffer.flip();
+ } else if (p >= 0) {
+ /*
+ * Requested position is after the current buffer limits and
+ * we can not seek, so we have to read everything before.
+ */
+ do {
+ bufferOffset += buffer.position();
+ buffer.clear();
+ final int c = channel.read(buffer);
+ if (c < 0) {
+ throw new EOFException(eof());
}
- variable.gridGeometry = gridGeometry;
- }
- gridGeometries = dimsToGG.values().toArray(new GridGeometry[dimsToGG.size()]);
+ p -= c;
+ } while (p > buffer.limit());
+ buffer.flip().position((int) p);
+ } else {
+ /*
+ * Requested position is before the current buffer limits
+ * and we can not seek.
+ */
+ throw new IOException(Errors.format(Errors.Keys.StreamIsForwardOnly_1, filename));
}
- return gridGeometries;
}
/**
- * Closes the channel.
- *
- * @throws IOException If an error occurred while closing the channel.
+ * Returns a string representation of this object for debugging purpose.
*/
+ @Debug
@Override
- public void close() throws IOException {
- channel.close();
+ public String toString() {
+ return getClass().getSimpleName() + "[“" + filename + "” at " + bufferOffset + buffer.position() + ']';
}
}
Added: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/package-info.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/package-info.java?rev=1485856&view=auto
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/package-info.java (added)
+++ sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/package-info.java [UTF-8] Thu May 23 21:26:19 2013
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utility classes for the implementation of SIS storage classes.
+ *
+ * <STRONG>Do not use!</STRONG>
+ *
+ * This package is for internal use by SIS only. Classes in this package
+ * may change in incompatible ways in any future version without notice.
+ *
+ * @author Martin Desruisseaux (Geomatys)
+ * @since 0.3
+ * @version 0.3
+ * @module
+ */
+package org.apache.sis.internal.storage;
Propchange: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/package-info.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: sis/branches/JDK7/storage/sis-netcdf/src/main/java/org/apache/sis/internal/storage/package-info.java
------------------------------------------------------------------------------
svn:mime-type = text/plain;charset=UTF-8
Added: sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/DataInputChannelCompleted.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/DataInputChannelCompleted.java?rev=1485856&view=auto
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/DataInputChannelCompleted.java (added)
+++ sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/DataInputChannelCompleted.java [UTF-8] Thu May 23 21:26:19 2013
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.internal.storage;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.ReadableByteChannel;
+
+
+/**
+ * Adds the missing methods in {@code DataInputChannel} for implementing the {@code DataInput} interface.
+ * This class is provided for testing the compatibility of {@code DataInputChannel} API with {@code DataInput},
+ * and as a placeholder in case we want to move this implementation in the main code in a future SIS version.
+ *
+ * @author Martin Desruisseaux (Geomatys)
+ * @since 0.3 (derived from geotk-3.07)
+ * @version 0.3
+ * @module
+ */
+public class DataInputChannelCompleted extends DataInputChannel implements DataInput {
+ /**
+ * Creates a new input source for the given channel and using the given buffer.
+ *
+ * @param filename A file identifier used only for formatting error message.
+ * @param channel The channel from where data are read.
+ * @param buffer The buffer where to copy the data.
+ * @param filled {@code true} if the buffer already contains data, or {@code false} if it needs
+ * to be initially filled with some content read from the channel.
+ * @throws IOException If an error occurred while reading the channel.
+ */
+ public DataInputChannelCompleted(final String filename, final ReadableByteChannel channel,
+ final ByteBuffer buffer, final boolean filled) throws IOException
+ {
+ super(filename, channel, buffer, filled);
+ }
+
+ /**
+ * Returns the next byte from the stream as an unsigned integer between 0 and 255, or -1 if
+ * we reached the end of stream.
+ *
+ * @return The next byte as an unsigned integer, or -1 on end of stream.
+ * @throws IOException If an error occurred while reading the stream.
+ */
+ public final int read() throws IOException {
+ return hasRemaining() ? buffer.get() & 0xFF : -1;
+ }
+
+ /**
+ * Reads a byte from the stream and returns a {@code true} if it is nonzero, {@code false} otherwise.
+ * The implementation is as below:
+ *
+ * {@preformat java
+ * return readByte() != 0;
+ * }
+ *
+ * @return The value of the next boolean from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
+ */
+ @Override
+ public final boolean readBoolean() throws IOException {
+ return readByte() != 0;
+ }
+
+ /**
+ * Skips over n bytes of data from the input stream.
+ * This implementation does not skip more bytes than the buffer capacity.
+ *
+ * @param n Maximal number of bytes to skip.
+ * @return Number of bytes actually skipped.
+ * @throws IOException If an error occurred while reading.
+ */
+ @Override
+ public final int skipBytes(int n) throws IOException {
+ if (!hasRemaining()) {
+ return 0;
+ }
+ int r = buffer.remaining();
+ if (n >= r) {
+ n = r;
+ }
+ buffer.position(buffer.position() + n);
+ return n;
+ }
+
+ /**
+ * Reads the new bytes until the next EOL. This method can read only US-ASCII strings.
+ * This method is provided for compliance with the {@link DataInput} interface,
+ * but is generally not recommended.
+ *
+ * @return The next line, or {@code null} if the EOF has been reached.
+ * @throws IOException If an error occurred while reading.
+ */
+ @Override
+ public final String readLine() throws IOException {
+ int c = read();
+ if (c < 0) {
+ return null;
+ }
+ StringBuilder line = new StringBuilder();
+ line.append((char) c);
+loop: while ((c = read()) >= 0) {
+ switch (c) {
+ case '\r': {
+ c = read();
+ if (c >= 0 && c != '\n') {
+ buffer.position(buffer.position() - 1);
+ }
+ break loop;
+ }
+ case '\n': {
+ break loop;
+ }
+ }
+ line.append((char) c);
+ }
+ return line.toString();
+ }
+
+ /**
+ * Reads in a string that has been encoded using a UTF-8 string.
+ *
+ * @return The string reads from the stream.
+ * @throws IOException If an error (including EOF) occurred while reading the stream.
+ */
+ @Override
+ public final String readUTF() throws IOException {
+ final ByteOrder oldOrder = buffer.order();
+ buffer.order(ByteOrder.BIG_ENDIAN);
+ try {
+ return DataInputStream.readUTF(this);
+ } finally {
+ buffer.order(oldOrder);
+ }
+ }
+}
Propchange: sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/DataInputChannelCompleted.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/internal/storage/DataInputChannelCompleted.java
------------------------------------------------------------------------------
svn:mime-type = text/plain;charset=UTF-8
Modified: sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/test/suite/NetcdfTestSuite.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/test/suite/NetcdfTestSuite.java?rev=1485856&r1=1485855&r2=1485856&view=diff
==============================================================================
--- sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/test/suite/NetcdfTestSuite.java [UTF-8] (original)
+++ sis/branches/JDK7/storage/sis-netcdf/src/test/java/org/apache/sis/test/suite/NetcdfTestSuite.java [UTF-8] Thu May 23 21:26:19 2013
@@ -35,7 +35,8 @@ import org.junit.BeforeClass;
org.apache.sis.internal.netcdf.VariableTest.class,
org.apache.sis.internal.netcdf.GridGeometryTest.class,
org.apache.sis.internal.netcdf.impl.ChannelDecoderTest.class,
- org.apache.sis.internal.netcdf.impl.VariableInfoTest.class
+ org.apache.sis.internal.netcdf.impl.VariableInfoTest.class,
+ org.apache.sis.internal.netcdf.impl.GridGeometryInfoTest.class
})
public final strictfp class NetcdfTestSuite extends TestSuite {
/**