You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@directmemory.apache.org by no...@apache.org on 2012/11/02 20:47:31 UTC
svn commit: r1405126 - in
/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning:
./ internal/util/ io/
Author: noctarius
Date: Fri Nov 2 19:47:31 2012
New Revision: 1405126
URL: http://svn.apache.org/viewvc?rev=1405126&view=rev
Log:
Merge branch 'trunk' of https://github.com/noctarius/directmemory-lightning into trunk
Added:
directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/internal/util/UnicodeUtil.java
Modified:
directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Source.java
directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Target.java
directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferSource.java
directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferTarget.java
directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/InputStreamSource.java
directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/OutputStreamTarget.java
Modified: directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Source.java
URL: http://svn.apache.org/viewvc/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Source.java?rev=1405126&r1=1405125&r2=1405126&view=diff
==============================================================================
--- directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Source.java (original)
+++ directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Source.java Fri Nov 2 19:47:31 2012
@@ -37,6 +37,9 @@ public interface Source
int readBytes( byte[] bytes, int offset, int length )
throws IOException;
+ boolean readBoolean()
+ throws IOException;
+
byte readByte()
throws IOException;
@@ -61,6 +64,9 @@ public interface Source
double readDouble()
throws IOException;
+ String readString()
+ throws IOException;
+
void clear()
throws IOException;
Modified: directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Target.java
URL: http://svn.apache.org/viewvc/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Target.java?rev=1405126&r1=1405125&r2=1405126&view=diff
==============================================================================
--- directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Target.java (original)
+++ directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/Target.java Fri Nov 2 19:47:31 2012
@@ -37,6 +37,9 @@ public interface Target
void writeBytes( byte[] bytes, int offset, int length )
throws IOException;
+ void writeBoolean( boolean value )
+ throws IOException;
+
void writeByte( byte value )
throws IOException;
@@ -61,6 +64,9 @@ public interface Target
void writeDouble( double value )
throws IOException;
+ void writeString( String value )
+ throws IOException;
+
void clear()
throws IOException;
Added: directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/internal/util/UnicodeUtil.java
URL: http://svn.apache.org/viewvc/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/internal/util/UnicodeUtil.java?rev=1405126&view=auto
==============================================================================
--- directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/internal/util/UnicodeUtil.java (added)
+++ directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/internal/util/UnicodeUtil.java Fri Nov 2 19:47:31 2012
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.directmemory.lightning.internal.util;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+
+import org.apache.directmemory.lightning.Source;
+import org.apache.directmemory.lightning.Target;
+
+/*
+ * This codebase is derived from the org.apache.lucene.util.UnicodeUtil class from Apache Lucene project.
+ */
+
+/*
+ * Some of this code came from the excellent Unicode
+ * conversion examples from:
+ *
+ * http://www.unicode.org/Public/PROGRAMS/CVTUTF
+ *
+ * Full Copyright for that code follows:
+ */
+
+/*
+ * Copyright 2001-2004 Unicode, Inc.
+ *
+ * Disclaimer
+ *
+ * This source code is provided as is by Unicode, Inc. No claims are
+ * made as to fitness for any particular purpose. No warranties of any
+ * kind are expressed or implied. The recipient agrees to determine
+ * applicability of information provided. If this file has been
+ * purchased on magnetic or optical media from Unicode, Inc., the
+ * sole remedy for any claim will be exchange of defective media
+ * within 90 days of receipt.
+ *
+ * Limitations on Rights to Redistribute This Code
+ *
+ * Unicode, Inc. hereby grants the right to freely use the information
+ * supplied in this file in the creation of products supporting the
+ * Unicode Standard, and to make copies of this file in any form
+ * for internal or external distribution as long as this notice
+ * remains attached.
+ */
+
+/*
+ * Additional code came from the IBM ICU library.
+ *
+ * http://www.icu-project.org
+ *
+ * Full Copyright for that code follows.
+ */
+
+/*
+ * Copyright (C) 1999-2010, International Business Machines
+ * Corporation and others. All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * provided that the above copyright notice(s) and this permission notice appear
+ * in all copies of the Software and that both the above copyright notice(s) and
+ * this permission notice appear in supporting documentation.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE
+ * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
+ * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+ * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Except as contained in this notice, the name of a copyright holder shall not
+ * be used in advertising or otherwise to promote the sale, use or other
+ * dealings in this Software without prior written authorization of the
+ * copyright holder.
+ */
+
+/**
+ * Class to encode java's UTF16 char[] into UTF8 byte[] without always allocating a new byte[] as
+ * String.getBytes("UTF-8") does.
+ */
+public final class UnicodeUtil
+{
+
+ private UnicodeUtil()
+ {
+ } // no instance
+
+ public static final int UNI_SUR_HIGH_START = 0xD800;
+
+ public static final int UNI_SUR_HIGH_END = 0xDBFF;
+
+ public static final int UNI_SUR_LOW_START = 0xDC00;
+
+ public static final int UNI_SUR_LOW_END = 0xDFFF;
+
+ public static final int UNI_REPLACEMENT_CHAR = 0xFFFD;
+
+ private static final long UNI_MAX_BMP = 0x0000FFFF;
+
+ private static final long HALF_SHIFT = 10;
+
+ private static final long HALF_MASK = 0x3FFL;
+
+ private static final int SURROGATE_OFFSET = Character.MIN_SUPPLEMENTARY_CODE_POINT
+ - ( UNI_SUR_HIGH_START << HALF_SHIFT ) - UNI_SUR_LOW_START;
+
+ // Special String package private internal constructor for sharing char-array usage
+ private static final Constructor<String> STRING_PP_CONSTRUCTOR;
+
+ static
+ {
+ Constructor<String> constructor = null;
+ try
+ {
+ constructor = String.class.getDeclaredConstructor( int.class, int.class, char[].class );
+ constructor.setAccessible( true );
+ }
+ catch ( SecurityException e )
+ {
+ // intentionally left blank
+ }
+ catch ( NoSuchMethodException e )
+ {
+ // intentionally left blank
+ }
+ STRING_PP_CONSTRUCTOR = constructor;
+ }
+
+ /**
+ * Encode characters from the given {@link String}, starting at offset 0 for length chars. Returns length of the
+ * encoded String in bytes.
+ *
+ * @throws IOException
+ */
+ public static int UTF16toUTF8( String value, Target target )
+ throws IOException
+ {
+ char[] characters = value.toCharArray();
+ int length = characters.length;
+
+ // Write string length to target
+ target.writeInt( length );
+
+ int i = 0;
+ final int end = length;
+
+ int writtenBytes = 0;
+ while ( i < end )
+ {
+
+ final int code = (int) characters[i++];
+
+ if ( code < 0x80 )
+ {
+ target.writeByte( (byte) code );
+ writtenBytes++;
+ }
+ else if ( code < 0x800 )
+ {
+ target.writeByte( (byte) ( 0xC0 | ( code >> 6 ) ) );
+ target.writeByte( (byte) ( 0x80 | ( code & 0x3F ) ) );
+ writtenBytes += 2;
+ }
+ else if ( code < 0xD800 || code > 0xDFFF )
+ {
+ target.writeByte( (byte) ( 0xE0 | ( code >> 12 ) ) );
+ target.writeByte( (byte) ( 0x80 | ( ( code >> 6 ) & 0x3F ) ) );
+ target.writeByte( (byte) ( 0x80 | ( code & 0x3F ) ) );
+ writtenBytes += 3;
+ }
+ else
+ {
+ // surrogate pair
+ // confirm valid high surrogate
+ if ( code < 0xDC00 && i < end )
+ {
+ int utf32 = (int) characters[i];
+ // confirm valid low surrogate and write pair
+ if ( utf32 >= 0xDC00 && utf32 <= 0xDFFF )
+ {
+ utf32 = ( code << 10 ) + utf32 + SURROGATE_OFFSET;
+ i++;
+ target.writeByte( (byte) ( 0xF0 | ( utf32 >> 18 ) ) );
+ target.writeByte( (byte) ( 0x80 | ( ( utf32 >> 12 ) & 0x3F ) ) );
+ target.writeByte( (byte) ( 0x80 | ( ( utf32 >> 6 ) & 0x3F ) ) );
+ target.writeByte( (byte) ( 0x80 | ( utf32 & 0x3F ) ) );
+ writtenBytes += 4;
+ continue;
+ }
+ }
+ // replace unpaired surrogate or out-of-order low surrogate
+ // with substitution character
+ target.writeByte( (byte) 0xEF );
+ target.writeByte( (byte) 0xBF );
+ target.writeByte( (byte) 0xBD );
+ writtenBytes += 3;
+ }
+ }
+ return writtenBytes;
+ }
+
+ public static boolean validUTF16String( CharSequence s )
+ {
+ final int size = s.length();
+ for ( int i = 0; i < size; i++ )
+ {
+ char ch = s.charAt( i );
+ if ( ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END )
+ {
+ if ( i < size - 1 )
+ {
+ i++;
+ char nextCH = s.charAt( i );
+ if ( nextCH >= UNI_SUR_LOW_START && nextCH <= UNI_SUR_LOW_END )
+ {
+ // Valid surrogate pair
+ }
+ else
+ // Unmatched high surrogate
+ return false;
+ }
+ else
+ // Unmatched high surrogate
+ return false;
+ }
+ else if ( ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END )
+ // Unmatched low surrogate
+ return false;
+ }
+
+ return true;
+ }
+
+ public static boolean validUTF16String( char[] s, int size )
+ {
+ for ( int i = 0; i < size; i++ )
+ {
+ char ch = s[i];
+ if ( ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END )
+ {
+ if ( i < size - 1 )
+ {
+ i++;
+ char nextCH = s[i];
+ if ( nextCH >= UNI_SUR_LOW_START && nextCH <= UNI_SUR_LOW_END )
+ {
+ // Valid surrogate pair
+ }
+ else
+ return false;
+ }
+ else
+ return false;
+ }
+ else if ( ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END )
+ // Unmatched low surrogate
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Interprets the bytes from the given {@link Source} as UTF-8 and converts to UTF-16.
+ * <p>
+ * NOTE: Full characters are read, even if this reads past the length passed (and can result in an
+ * ArrayOutOfBoundsException if invalid UTF-8 is passed). Explicit checks for valid UTF-8 are not performed.
+ *
+ * @throws IOException
+ */
+ public static String UTF8toUTF16( Source source, int length )
+ throws IOException
+ {
+ int charLength = source.readInt();
+
+ int offset = 0;
+ final char[] out = new char[charLength];
+ while ( offset < length )
+ {
+ int b = source.readByte() & 0xff;
+ if ( b < 0xc0 )
+ {
+ assert b < 0x80;
+ out[offset++] = (char) b;
+ }
+ else if ( b < 0xe0 )
+ {
+ out[offset++] = (char) ( ( ( b & 0x1f ) << 6 ) + ( source.readByte() & 0x3f ) );
+ }
+ else if ( b < 0xf0 )
+ {
+ out[offset++] =
+ (char) ( ( ( b & 0xf ) << 12 ) + ( ( source.readByte() & 0x3f ) << 6 ) + ( source.readByte() & 0x3f ) );
+ }
+ else
+ {
+ assert b < 0xf8 : "b = 0x" + Integer.toHexString( b );
+ int ch =
+ ( ( b & 0x7 ) << 18 ) + ( ( source.readByte() & 0x3f ) << 12 )
+ + ( ( source.readByte() & 0x3f ) << 6 ) + ( source.readByte() & 0x3f );
+ if ( ch < UNI_MAX_BMP )
+ {
+ out[offset++] = (char) ch;
+ }
+ else
+ {
+ int chHalf = ch - 0x0010000;
+ out[offset++] = (char) ( ( chHalf >> 10 ) + 0xD800 );
+ out[offset++] = (char) ( ( chHalf & HALF_MASK ) + 0xDC00 );
+ }
+ }
+ }
+
+ if ( STRING_PP_CONSTRUCTOR == null )
+ {
+ return new String( out );
+ }
+ else
+ {
+ try
+ {
+ return STRING_PP_CONSTRUCTOR.newInstance( 0, out.length, out );
+ }
+ catch ( Exception e )
+ {
+ return new String( out );
+ }
+ }
+ }
+
+}
Modified: directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferSource.java
URL: http://svn.apache.org/viewvc/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferSource.java?rev=1405126&r1=1405125&r2=1405126&view=diff
==============================================================================
--- directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferSource.java (original)
+++ directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferSource.java Fri Nov 2 19:47:31 2012
@@ -5,6 +5,7 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.apache.directmemory.lightning.Source;
+import org.apache.directmemory.lightning.internal.util.UnicodeUtil;
public class ByteBufferSource
implements Source
@@ -52,6 +53,21 @@ public class ByteBufferSource
}
@Override
+ public boolean readBoolean()
+ throws IOException
+ {
+ return readByte() == 1 ? true : false;
+ }
+
+ @Override
+ public String readString()
+ throws IOException
+ {
+ int length = readInt();
+ return UnicodeUtil.UTF8toUTF16( this, length );
+ }
+
+ @Override
public byte readByte()
throws IOException
{
Modified: directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferTarget.java
URL: http://svn.apache.org/viewvc/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferTarget.java?rev=1405126&r1=1405125&r2=1405126&view=diff
==============================================================================
--- directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferTarget.java (original)
+++ directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/ByteBufferTarget.java Fri Nov 2 19:47:31 2012
@@ -51,6 +51,20 @@ public class ByteBufferTarget
}
@Override
+ public void writeBoolean( boolean value )
+ throws IOException
+ {
+ writeByte( (byte) ( value ? 1 : 0 ) );
+ }
+
+ @Override
+ public void writeString( String value )
+ throws IOException
+ {
+ // TODO: implementation of writeString is missing
+ }
+
+ @Override
public void writeByte( byte value )
throws IOException
{
Modified: directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/InputStreamSource.java
URL: http://svn.apache.org/viewvc/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/InputStreamSource.java?rev=1405126&r1=1405125&r2=1405126&view=diff
==============================================================================
--- directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/InputStreamSource.java (original)
+++ directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/InputStreamSource.java Fri Nov 2 19:47:31 2012
@@ -24,6 +24,7 @@ import java.nio.ByteOrder;
import org.apache.directmemory.lightning.Source;
import org.apache.directmemory.lightning.internal.io.ByteOrderUtils;
+import org.apache.directmemory.lightning.internal.util.UnicodeUtil;
public class InputStreamSource
implements Source
@@ -85,6 +86,21 @@ public class InputStreamSource
}
@Override
+ public boolean readBoolean()
+ throws IOException
+ {
+ return readByte() == 1 ? true : false;
+ }
+
+ @Override
+ public String readString()
+ throws IOException
+ {
+ int length = readInt();
+ return UnicodeUtil.UTF8toUTF16( this, length );
+ }
+
+ @Override
public byte readByte()
throws IOException
{
Modified: directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/OutputStreamTarget.java
URL: http://svn.apache.org/viewvc/directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/OutputStreamTarget.java?rev=1405126&r1=1405125&r2=1405126&view=diff
==============================================================================
--- directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/OutputStreamTarget.java (original)
+++ directmemory/lightning/trunk/lightning-core/src/main/java/org/apache/directmemory/lightning/io/OutputStreamTarget.java Fri Nov 2 19:47:31 2012
@@ -63,6 +63,20 @@ public class OutputStreamTarget
}
@Override
+ public void writeBoolean( boolean value )
+ throws IOException
+ {
+ writeByte( (byte) ( value ? 1 : 0 ) );
+ }
+
+ @Override
+ public void writeString( String value )
+ throws IOException
+ {
+ // TODO: implementation of writeString is missing
+ }
+
+ @Override
public void writeByte( byte value )
throws IOException
{