You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@tomcat.apache.org by ma...@apache.org on 2019/12/06 19:42:57 UTC
[tomcat] 02/09: Merge in Codec changes to 9637dd4 (2019-12-06,
1.14-SNAPSHOT)
This is an automated email from the ASF dual-hosted git repository.
markt pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tomcat.git
commit 98ef428f5c5b5524582c6c440f16a7e3e9562ba8
Author: Mark Thomas <ma...@apache.org>
AuthorDate: Fri Dec 6 14:29:01 2019 +0000
Merge in Codec changes to 9637dd4 (2019-12-06, 1.14-SNAPSHOT)
---
MERGE.txt | 2 +-
.../apache/tomcat/util/codec/binary/Base64.java | 28 +++---
.../tomcat/util/codec/binary/BaseNCodec.java | 100 +++++++++++++++++----
webapps/docs/changelog.xml | 4 +
4 files changed, 107 insertions(+), 27 deletions(-)
diff --git a/MERGE.txt b/MERGE.txt
index f4fb1f8..b4bd507 100644
--- a/MERGE.txt
+++ b/MERGE.txt
@@ -43,7 +43,7 @@ Codec
Sub-tree:
src/main/java/org/apache/commons/codec
The SHA1 ID for the most recent commit to be merged to Tomcat is:
-3ebef4ad92e31697fb52ca7cc71904c68654c2c8 (2019-08-01)
+9637dd44fa0e2d5a6ddb45791e3cd78298842d95 (2019-12-06)
Note: Only classes required for Base64 encoding/decoding. The rest are removed.
FileUpload
diff --git a/java/org/apache/tomcat/util/codec/binary/Base64.java b/java/org/apache/tomcat/util/codec/binary/Base64.java
index da1487f..ab89854 100644
--- a/java/org/apache/tomcat/util/codec/binary/Base64.java
+++ b/java/org/apache/tomcat/util/codec/binary/Base64.java
@@ -139,6 +139,10 @@ public class Base64 extends BaseNCodec {
*/
/** Mask used to extract 6 bits, used when encoding */
private static final int MASK_6BITS = 0x3f;
+ /** Mask used to extract 4 bits, used when decoding final trailing character. */
+ private static final int MASK_4BITS = 0xf;
+ /** Mask used to extract 2 bits, used when decoding final trailing character. */
+ private static final int MASK_2BITS = 0x3;
// The static final fields above are used for the original static byte[] methods on Base64.
// The private member fields below are used with the new streaming approach, which requires
@@ -483,12 +487,12 @@ public class Base64 extends BaseNCodec {
// TODO not currently tested; perhaps it is impossible?
break;
case 2 : // 12 bits = 8 + 4
- validateCharacter(4, context);
+ validateCharacter(MASK_4BITS, context);
context.ibitWorkArea = context.ibitWorkArea >> 4; // dump the extra 4 bits
buffer[context.pos++] = (byte) ((context.ibitWorkArea) & MASK_8BITS);
break;
case 3 : // 18 bits = 8 + 8 + 2
- validateCharacter(2, context);
+ validateCharacter(MASK_2BITS, context);
context.ibitWorkArea = context.ibitWorkArea >> 2; // dump 2 bits
buffer[context.pos++] = (byte) ((context.ibitWorkArea >> 8) & MASK_8BITS);
buffer[context.pos++] = (byte) ((context.ibitWorkArea) & MASK_8BITS);
@@ -792,20 +796,22 @@ public class Base64 extends BaseNCodec {
/**
- * <p>
- * Validates whether the character is possible in the context of the set of possible base 64 values.
- * </p>
+ * Validates whether decoding the final trailing character is possible in the context
+ * of the set of possible base 64 values.
+ *
+ * <p>The character is valid if the lower bits within the provided mask are zero. This
+ * is used to test the final trailing base-64 digit is zero in the bits that will be discarded.
*
- * @param numBitsToDrop number of least significant bits to check
+ * @param emptyBitsMask The mask of the lower bits that should be empty
* @param context the context to be used
*
* @throws IllegalArgumentException if the bits being checked contain any non-zero value
*/
- private long validateCharacter(final int numBitsToDrop, final Context context) {
- if ((context.ibitWorkArea & numBitsToDrop) != 0) {
- throw new IllegalArgumentException(
- "Last encoded character (before the paddings if any) is a valid base 64 alphabet but not a possible value");
+ private static void validateCharacter(final int emptyBitsMask, final Context context) {
+ if ((context.ibitWorkArea & emptyBitsMask) != 0) {
+ throw new IllegalArgumentException(
+ "Last encoded character (before the paddings if any) is a valid base 64 alphabet but not a possible value. " +
+ "Expected the discarded bits to be zero.");
}
- return context.ibitWorkArea >> numBitsToDrop;
}
}
diff --git a/java/org/apache/tomcat/util/codec/binary/BaseNCodec.java b/java/org/apache/tomcat/util/codec/binary/BaseNCodec.java
index d24b43c..83e81ba 100644
--- a/java/org/apache/tomcat/util/codec/binary/BaseNCodec.java
+++ b/java/org/apache/tomcat/util/codec/binary/BaseNCodec.java
@@ -136,6 +136,18 @@ public abstract class BaseNCodec {
*/
private static final int DEFAULT_BUFFER_SIZE = 128;
+ /**
+ * The maximum size buffer to allocate.
+ *
+ * <p>This is set to the same size used in the JDK {@code java.util.ArrayList}:</p>
+ * <blockquote>
+ * Some VMs reserve some header words in an array.
+ * Attempts to allocate larger arrays may result in
+ * OutOfMemoryError: Requested array size exceeds VM limit.
+ * </blockquote>
+ */
+ private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8;
+
/** Mask used to extract 8 bits, used in decoding bytes */
protected static final int MASK_8BITS = 0xff;
@@ -165,7 +177,7 @@ public abstract class BaseNCodec {
private final int chunkSeparatorLength;
/**
- * Note <code>lineLength</code> is rounded down to the nearest multiple of {@link #encodedBlockSize}
+ * Note <code>lineLength</code> is rounded down to the nearest multiple of the encoded block size.
* If <code>chunkSeparatorLength</code> is zero, then chunking is disabled.
* @param unencodedBlockSize the size of an unencoded block (e.g. Base64 = 3)
* @param encodedBlockSize the size of an encoded block (e.g. Base64 = 4)
@@ -178,7 +190,7 @@ public abstract class BaseNCodec {
}
/**
- * Note <code>lineLength</code> is rounded down to the nearest multiple of {@link #encodedBlockSize}
+ * Note <code>lineLength</code> is rounded down to the nearest multiple of the encoded block size.
* If <code>chunkSeparatorLength</code> is zero, then chunking is disabled.
* @param unencodedBlockSize the size of an unencoded block (e.g. Base64 = 3)
* @param encodedBlockSize the size of an encoded block (e.g. Base64 = 4)
@@ -220,7 +232,7 @@ public abstract class BaseNCodec {
/**
* Get the default buffer size. Can be overridden.
*
- * @return {@link #DEFAULT_BUFFER_SIZE}
+ * @return the default buffer size.
*/
protected int getDefaultBufferSize() {
return DEFAULT_BUFFER_SIZE;
@@ -229,18 +241,69 @@ public abstract class BaseNCodec {
/**
* Increases our buffer by the {@link #DEFAULT_BUFFER_RESIZE_FACTOR}.
* @param context the context to be used
+ * @param minCapacity the minimum required capacity
+ * @return the resized byte[] buffer
+ * @throws OutOfMemoryError if the {@code minCapacity} is negative
*/
- private byte[] resizeBuffer(final Context context) {
- if (context.buffer == null) {
- context.buffer = new byte[getDefaultBufferSize()];
- context.pos = 0;
- context.readPos = 0;
- } else {
- final byte[] b = new byte[context.buffer.length * DEFAULT_BUFFER_RESIZE_FACTOR];
- System.arraycopy(context.buffer, 0, b, 0, context.buffer.length);
- context.buffer = b;
+ private static byte[] resizeBuffer(final Context context, final int minCapacity) {
+ // Overflow-conscious code treats the min and new capacity as unsigned.
+ final int oldCapacity = context.buffer.length;
+ int newCapacity = oldCapacity * DEFAULT_BUFFER_RESIZE_FACTOR;
+ if (compareUnsigned(newCapacity, minCapacity) < 0) {
+ newCapacity = minCapacity;
}
- return context.buffer;
+ if (compareUnsigned(newCapacity, MAX_BUFFER_SIZE) > 0) {
+ newCapacity = createPositiveCapacity(minCapacity);
+ }
+
+ final byte[] b = new byte[newCapacity];
+ System.arraycopy(context.buffer, 0, b, 0, context.buffer.length);
+ context.buffer = b;
+ return b;
+ }
+
+ /**
+ * Compares two {@code int} values numerically treating the values
+ * as unsigned. Taken from JDK 1.8.
+ *
+ * <p>TODO: Replace with JDK 1.8 Integer::compareUnsigned(int, int).</p>
+ *
+ * @param x the first {@code int} to compare
+ * @param y the second {@code int} to compare
+ * @return the value {@code 0} if {@code x == y}; a value less
+ * than {@code 0} if {@code x < y} as unsigned values; and
+ * a value greater than {@code 0} if {@code x > y} as
+ * unsigned values
+ */
+ private static int compareUnsigned(int x, int y) {
+ return Integer.compare(x + Integer.MIN_VALUE, y + Integer.MIN_VALUE);
+ }
+
+ /**
+ * Create a positive capacity at least as large the minimum required capacity.
+ * If the minimum capacity is negative then this throws an OutOfMemoryError as no array
+ * can be allocated.
+ *
+ * @param minCapacity the minimum capacity
+ * @return the capacity
+ * @throws OutOfMemoryError if the {@code minCapacity} is negative
+ */
+ private static int createPositiveCapacity(int minCapacity) {
+ if (minCapacity < 0) {
+ // overflow
+ throw new OutOfMemoryError("Unable to allocate array size: " + (minCapacity & 0xffffffffL));
+ }
+ // This is called when we require buffer expansion to a very big array.
+ // Use the conservative maximum buffer size if possible, otherwise the biggest required.
+ //
+ // Note: In this situation JDK 1.8 java.util.ArrayList returns Integer.MAX_VALUE.
+ // This excludes some VMs that can exceed MAX_BUFFER_SIZE but not allocate a full
+ // Integer.MAX_VALUE length array.
+ // The result is that we may have to allocate an array of this size more than once if
+ // the capacity must be expanded again.
+ return (minCapacity > MAX_BUFFER_SIZE) ?
+ minCapacity :
+ MAX_BUFFER_SIZE;
}
/**
@@ -251,8 +314,15 @@ public abstract class BaseNCodec {
* @return the buffer
*/
protected byte[] ensureBufferSize(final int size, final Context context){
- if ((context.buffer == null) || (context.buffer.length < context.pos + size)){
- return resizeBuffer(context);
+ if (context.buffer == null) {
+ context.buffer = new byte[getDefaultBufferSize()];
+ context.pos = 0;
+ context.readPos = 0;
+
+ // Overflow-conscious:
+ // x + y > z == x + y - z > 0
+ } else if (context.pos + size - context.buffer.length > 0) {
+ return resizeBuffer(context, context.pos + size);
}
return context.buffer;
}
diff --git a/webapps/docs/changelog.xml b/webapps/docs/changelog.xml
index abb85c0..d93a21c 100644
--- a/webapps/docs/changelog.xml
+++ b/webapps/docs/changelog.xml
@@ -208,6 +208,10 @@
Update the internal fork of Apache Commons BCEL to ff6941e (2019-12-06,
6.4.2-dev). Code clean-up only. (markt)
</add>
+ <add>
+ Update the internal fork of Apache Commons Codec to 9637dd4 (2019-12-06,
+ 1.14-SNAPSHOT). Code clean-up and a fix for CODEC-265. (markt)
+ </add>
</changelog>
</subsection>
</section>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@tomcat.apache.org
For additional commands, e-mail: dev-help@tomcat.apache.org