You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@abdera.apache.org by jm...@apache.org on 2008/01/01 05:59:47 UTC
svn commit: r607801 [5/5] - in /incubator/abdera/java/trunk:
client/src/main/java/org/apache/abdera/protocol/client/
core/src/main/java/org/apache/abdera/util/
dependencies/i18n/src/main/java/org/apache/abdera/i18n/io/
dependencies/i18n/src/main/java/o...
Added: incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/CharsetSniffingInputStream.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/CharsetSniffingInputStream.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/CharsetSniffingInputStream.java (added)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/CharsetSniffingInputStream.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,159 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+package org.apache.abdera.i18n.text.io;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Will attempt to autodetect the character encoding from the stream
+ * By default, this will preserve the BOM if it exists
+ */
+public class CharsetSniffingInputStream
+ extends FilterInputStream {
+
+ public static enum Encoding {
+ UTF32be(
+ "UTF-32",
+ true,
+ new byte[] {0x00,0x00,0xFFFFFFFE,0xFFFFFFFF}),
+ UTF32le(
+ "UTF-32",
+ true,
+ new byte[] {0xFFFFFFFF,0xFFFFFFFE,0x00,0x00}),
+ INVALID(
+ null,
+ true,
+ new byte[] {0xFFFFFFFE,0xFFFFFFFF,0x00,0x00},
+ new byte[] {0x00,0x00,0xFFFFFFFF,0xFFFFFFFE}),
+ UTF16be(
+ "UTF-16",
+ true,
+ new byte[] {0xFFFFFFFE,0xFFFFFFFF}),
+ UTF16le(
+ "UTF-16",
+ true,
+ new byte[] {0xFFFFFFFF,0xFFFFFFFE}),
+ UTF8(
+ "UTF-8",
+ true,
+ new byte[] {0xFFFFFFEF,0xFFFFFFBB,0xFFFFFFBF}),
+ UTF32be2(
+ "UTF-32be",
+ false,
+ new byte[] {0x00,0x00,0x00,0x3C}),
+ UTF32le2(
+ "UTF-32le",
+ false,
+ new byte[] {0x3C,0x00,0x00,0x00}),
+ UTF16be2(
+ "UTF-16be",
+ false,
+ new byte[] {0x00,0x3C,0x00,0x3F}),
+ UTF16le2(
+ "UTF-16le",
+ false,
+ new byte[] {0x3C,0x00,0x3F,0x00})
+ ;
+
+ private final String enc;
+ private final byte[][] checks;
+ private final boolean bom;
+ Encoding(
+ String name,
+ boolean bom,
+ byte[]... checks) {
+ this.enc = name;
+ this.checks = checks;
+ this.bom = bom;
+ }
+ public String getEncoding() {
+ return enc;
+ }
+ public boolean getBom() {
+ return bom;
+ }
+ public int equals(byte[] bom) {
+ for (byte[] check : checks) {
+ if (CharsetSniffingInputStream.equals(bom, check.length, check))
+ return check.length;
+ }
+ return 0;
+ }
+ }
+
+ protected String encoding;
+ protected boolean bomset = false;
+ protected final boolean preserve;
+
+ public CharsetSniffingInputStream(InputStream in) {
+ this(in,true);
+ }
+
+ public CharsetSniffingInputStream(
+ InputStream in,
+ boolean preserveBom) {
+ super(
+ !(in instanceof PeekAheadInputStream) ?
+ new PeekAheadInputStream(in,4) : in);
+ this.preserve = preserveBom;
+ try {
+ encoding = detectEncoding();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public boolean isBomSet() {
+ return bomset;
+ }
+
+ public String getEncoding() {
+ return encoding;
+ }
+
+ protected PeekAheadInputStream getInternal() {
+ return (PeekAheadInputStream)in;
+ }
+
+ private static boolean equals(byte[] a1, int len, byte[] a2) {
+ for (int n = 0, i = 0; n < len; n++, i++) {
+ if (a1[n] != a2[i]) return false;
+ }
+ return true;
+ }
+
+ protected String detectEncoding() throws IOException {
+ PeekAheadInputStream pin = (PeekAheadInputStream) this.in;
+ byte[] bom = new byte[4];
+ pin.peek(bom);
+ bomset = false;
+ for (Encoding enc : Encoding.values()) {
+ int bomlen = enc.equals(bom);
+ if (bomlen > 0) {
+ bomset = enc.getBom();
+ if (bomset && !preserve) // consume the bom
+ pin.read(new byte[bomlen]);
+ return enc.getEncoding();
+ }
+ }
+ return null;
+ }
+
+}
Added: incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/DynamicPushbackInputStream.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/DynamicPushbackInputStream.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/DynamicPushbackInputStream.java (added)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/DynamicPushbackInputStream.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,125 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+package org.apache.abdera.i18n.text.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
+
+/**
+ * PushbackInputStream implementation that performs dynamic resizing of
+ * the unread buffer
+ */
+public class DynamicPushbackInputStream
+ extends PushbackInputStream {
+
+ private final int origsize;
+
+ public DynamicPushbackInputStream(
+ InputStream in) {
+ super(in);
+ this.origsize = 1;
+ }
+
+ public DynamicPushbackInputStream(
+ InputStream in,
+ int initialSize) {
+ super(in,initialSize);
+ this.origsize = initialSize;
+ }
+
+ /**
+ * Clear the buffer
+ */
+ public int clear() {
+ int m = buf.length;
+ buf = new byte[origsize];
+ pos = origsize;
+ return m;
+ }
+
+ /**
+ * Shrink the buffer. This will reclaim currently unused space in the
+ * buffer, reducing memory but potentially increasing the cost of
+ * resizing the buffer
+ */
+ public int shrink() {
+ byte[] old = buf;
+ if (pos == 0) return 0; // nothing to do
+ int n = old.length - pos;
+ int m, p,s,l;
+ if (n < origsize) {
+ buf = new byte[origsize];
+ p = pos;
+ s = origsize - n;
+ l = old.length-p;
+ m = old.length - origsize;
+ pos = s;
+ } else {
+ buf = new byte[n];
+ p = pos;
+ s = 0;
+ l = n;
+ m = old.length - l;
+ pos = 0;
+ }
+ System.arraycopy(old, p, buf, s, l);
+ return m;
+ }
+
+ private void resize(int len) {
+ byte[] old = buf;
+ buf = new byte[old.length + len];
+ System.arraycopy(old, 0, buf, len, old.length);
+ }
+
+ public void unread(byte[] b, int off, int len) throws IOException {
+ if (len > pos && pos + len > buf.length) {
+ resize(len-pos);
+ pos += len-pos;
+ }
+ super.unread(b, off, len);
+ }
+
+ public void unread(int b) throws IOException {
+ if (pos == 0) {
+ resize(1);
+ pos++;
+ }
+ super.unread(b);
+ }
+
+ public int read() throws IOException {
+ int m = super.read();
+ if (pos >= buf.length && buf.length > origsize) shrink();
+ return m;
+ }
+
+ public int read(byte[] b, int off, int len) throws IOException {
+ this.available(); // workaround for a problem in PushbackInputStream, without this, the amount of bytes read from some streams will be incorrect
+ int r = super.read(b, off, len);
+ if (pos >= buf.length && buf.length > origsize) shrink();
+ return r;
+ }
+
+ public long skip(long n) throws IOException {
+ long r = super.skip(n);
+ if (pos >= buf.length && buf.length > origsize) shrink();
+ return r;
+ }
+}
Added: incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/FilteredCharReader.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/FilteredCharReader.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/FilteredCharReader.java (added)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/FilteredCharReader.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,154 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+package org.apache.abdera.i18n.text.io;
+
+import java.io.FilterReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+
+import org.apache.abdera.i18n.text.Filter;
+
+
+/**
+ * A reader implementation that filters out unwanted characters
+ *
+ * By default, unwanted characters are simply removed from the stream.
+ * Alternatively, a replacement character can be provided so long as it
+ * is acceptable to the specified filter
+ */
+public class FilteredCharReader
+ extends FilterReader {
+
+ /**
+ * The XMLVersion determines which set of restrictions to apply depending
+ * on the XML version being parsed
+ */
+ private final Filter filter;
+ private final char replacement;
+
+ public FilteredCharReader(
+ InputStream in,
+ Filter filter) {
+ this(new InputStreamReader(in),filter);
+ }
+
+ public FilteredCharReader(
+ InputStream in,
+ String charset,
+ Filter filter)
+ throws UnsupportedEncodingException {
+ this(new InputStreamReader(in,charset),filter);
+ }
+
+ public FilteredCharReader(
+ InputStream in,
+ Filter filter,
+ char replacement) {
+ this(new InputStreamReader(in),filter,replacement);
+ }
+
+ public FilteredCharReader(
+ InputStream in,
+ String charset,
+ Filter filter,
+ char replacement)
+ throws UnsupportedEncodingException {
+ this(new InputStreamReader(in,charset),filter,replacement);
+ }
+
+ public FilteredCharReader(
+ Reader in) {
+ this(in,new NonOpFilter(),(char)0);
+ }
+
+ public FilteredCharReader(
+ Reader in,
+ Filter filter) {
+ this(in,filter,(char)0);
+ }
+
+ public FilteredCharReader(
+ Reader in,
+ char replacement) {
+ this(in,new NonOpFilter(),replacement);
+ }
+
+ public FilteredCharReader(
+ Reader in,
+ Filter filter,
+ char replacement) {
+ super(in);
+ this.filter = filter;
+ this.replacement = replacement;
+ if (replacement != 0 &&
+ ((!Character.isValidCodePoint(replacement)) ||
+ !filter.accept(replacement)))
+ throw new IllegalArgumentException();
+ }
+
+ @Override
+ public int read() throws IOException {
+ int c = -1;
+ if (replacement == 0) {
+ while(((c = super.read()) != -1 && !filter.accept(c))) {}
+ } else {
+ c = super.read();
+ if (c != -1 && !filter.accept(c)) c = replacement;
+ }
+ return c;
+ }
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ int n = off;
+ for (; n < Math.min(len,cbuf.length-off); n++) {
+ int r = read();
+ if (r != -1) cbuf[n] = (char)r;
+ else break;
+ }
+ n -= off;
+ return n <= 0 ? -1 : n;
+ }
+
+ public static Filter getUnacceptableFilter(int... unacceptable) {
+ return new CharArrayFilter(unacceptable);
+ }
+
+ private static class NonOpFilter
+ implements Filter {
+ public boolean accept(int c) {
+ return true;
+ }
+ }
+
+ private static class CharArrayFilter
+ implements Filter {
+ private final int[] chars;
+ public CharArrayFilter(int[] chars) {
+ this.chars = chars;
+ Arrays.sort(this.chars);
+ }
+ public boolean accept(int c) {
+ return Arrays.binarySearch(chars, c) < 0;
+ }
+ }
+}
Added: incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/InputStreamDataSource.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/InputStreamDataSource.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/InputStreamDataSource.java (added)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/InputStreamDataSource.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,64 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+package org.apache.abdera.i18n.text.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import javax.activation.DataSource;
+
+public final class InputStreamDataSource
+ implements DataSource {
+
+ public static final String DEFAULT_TYPE = "application/octet-stream";
+
+ private final InputStream in;
+ private final String ctype;
+
+ public InputStreamDataSource(
+ InputStream in) {
+ this(in,null);
+ }
+
+ public InputStreamDataSource(
+ InputStream in,
+ String ctype) {
+ this.in = in;
+ this.ctype = (ctype != null) ? ctype : DEFAULT_TYPE;
+ }
+
+ public String getContentType() {
+ return ctype;
+ }
+
+ public String getName() {
+ return null;
+ }
+
+ public InputStream getInputStream()
+ throws IOException {
+ return in;
+ }
+
+ public OutputStream getOutputStream()
+ throws IOException {
+ return null;
+ }
+
+}
\ No newline at end of file
Added: incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PeekAheadInputStream.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PeekAheadInputStream.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PeekAheadInputStream.java (added)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PeekAheadInputStream.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,75 @@
+package org.apache.abdera.i18n.text.io;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A version of RewindableInputStream that provides methods for peeking ahead
+ * in the stream (equivalent to read() followed by an appropriate unread()
+ */
+public class PeekAheadInputStream
+ extends RewindableInputStream {
+
+ public PeekAheadInputStream(
+ InputStream in) {
+ super(in);
+ }
+
+ public PeekAheadInputStream(
+ InputStream in,
+ int initialSize) {
+ super(in,initialSize);
+ }
+
+ /**
+ * Peek the next byte in the stream
+ */
+ public int peek()
+ throws IOException {
+ int m = read();
+ unread(m);
+ return m;
+ }
+
+ /**
+ * Peek the next bytes in the stream. Returns the number of bytes peeked.
+ * Will return -1 if the end of the stream is reached
+ */
+ public int peek(
+ byte[] buf)
+ throws IOException {
+ return peek(buf, 0, buf.length);
+ }
+
+ /**
+ * Peek the next bytes in the stream. Returns the number of bytes peeked.
+ * Will return -1 if the end of the stream is reached
+ */
+ public int peek(
+ byte[] buf,
+ int off,
+ int len)
+ throws IOException {
+ int r = read(buf, off, len);
+ unread(buf,off,len);
+ return r;
+ }
+
+}
Added: incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PipeChannel.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PipeChannel.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PipeChannel.java (added)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/PipeChannel.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,361 @@
+package org.apache.abdera.i18n.text.io;
+
+
+import java.io.Closeable;
+import java.io.FilterInputStream;
+import java.io.FilterOutputStream;
+import java.io.FilterReader;
+import java.io.FilterWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.Pipe;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.charset.Charset;
+
+import org.apache.abdera.i18n.text.CodepointIterator;
+
+
+/**
+ * Implements a buffer that provides a slightly more efficient way of writing,
+ * and then reading a stream of bytes.
+ *
+ * To use:
+ *
+ * PipeChannel pipe = new PipeChannel();
+ * byte[] bytes = {'a','b','c','d'};
+ * pipe.write(bytes);
+ * pipe.close();
+ *
+ * InputStream in = pipe.getInputStream();
+ * int i = -1;
+ * while ((i = in.read()) != -1) {...}
+ *
+ * By default, closing will automatically cause it to flip over to Read mode,
+ * locking the buffer from further writes and setting the read position to 0.
+ *
+ * Once the Buffer has been fully read, it must be reset, which sets it
+ * back into write mode
+ *
+ */
+public class PipeChannel
+ implements ReadableByteChannel,
+ WritableByteChannel,
+ Appendable,
+ Readable,
+ Closeable {
+
+ protected String charset = Charset.defaultCharset().name();
+
+ protected Pipe pipe;
+ protected boolean flipped = false;
+
+ public PipeChannel() {
+ reset();
+ }
+
+ public PipeChannel(String charset) {
+ this.charset = charset;
+ }
+
+ private void checkFlipped() {
+ if (flipped) throw new RuntimeException("PipeChannel is read only");
+ }
+
+ private void checkNotFlipped() {
+ if (!flipped) throw new RuntimeException("PipeChannel is write only");
+ }
+
+ /**
+ * Get an inputstream that can read from this pipe. The Pipe must be readable
+ */
+ public InputStream getInputStream() {
+ checkNotFlipped();
+ return new PipeChannelInputStream(this,Channels.newInputStream(pipe.source()));
+ }
+
+ /**
+ * Get an outputstream that can write to this pipe. The Pipe must be writable
+ */
+ public OutputStream getOutputStream() {
+ checkFlipped();
+ return new PipeChannelOutputStream(this,Channels.newOutputStream(pipe.sink()));
+ }
+
+ /**
+ * Get a writer that can write to this pipe. The pipe must be writable
+ */
+ public Writer getWriter() {
+ checkFlipped();
+ return new PipeChannelWriter(this,Channels.newWriter(pipe.sink(),charset));
+ }
+
+ /**
+ * Get a writer that can write to this pipe. The pipe must be writable
+ */
+ public Writer getWriter(String charset) {
+ checkFlipped();
+ return new PipeChannelWriter(this,Channels.newWriter(pipe.sink(), charset));
+ }
+
+ /**
+ * Get a reader that can reader from this pipe. The pipe must be readable
+ */
+ public Reader getReader(String charset) {
+ checkNotFlipped();
+ return new PipeChannelReader(this,Channels.newReader(pipe.source(), charset));
+ }
+
+ /**
+ * Get a reader that can reader from this pipe. The pipe must be readable
+ */
+ public Reader getReader() {
+ checkNotFlipped();
+ return new PipeChannelReader(this,Channels.newReader(pipe.source(), charset));
+ }
+
+ /**
+ * Get a CodepointIterator that can iterate over unicode codepoints in this pipe.
+ * The pipe must be readable
+ */
+ public CodepointIterator getIterator() {
+ checkNotFlipped();
+ return CodepointIterator.forReadableByteChannel(pipe.source(), charset);
+ }
+
+ /**
+ * Get a CodepointIterator that can iterate over unicode codepoints in this pipe.
+ * The pipe must be readable
+ */
+ public CodepointIterator getIterator(String charset) {
+ checkNotFlipped();
+ return CodepointIterator.forReadableByteChannel(pipe.source(), charset);
+ }
+
+ /**
+ * Read from the pipe.
+ */
+ public int read(ByteBuffer dst) throws IOException {
+ checkNotFlipped();
+ return pipe.source().read(dst);
+ }
+
+ /**
+ * Read from the pipe.
+ */
+ public int read(byte[] dst) throws IOException {
+ checkNotFlipped();
+ return pipe.source().read(ByteBuffer.wrap(dst));
+ }
+
+ /**
+ * Read from the pipe.
+ */
+ public int read(byte[] dst, int offset, int length) throws IOException {
+ checkNotFlipped();
+ return pipe.source().read(ByteBuffer.wrap(dst,offset,length));
+ }
+
+ /**
+ * True if the pipe is open
+ */
+ public boolean isOpen() {
+ return pipe.sink().isOpen() || pipe.source().isOpen();
+ }
+
+ /**
+ * Write to the pipe
+ */
+ public int write(ByteBuffer src) throws IOException {
+ checkFlipped();
+ return pipe.sink().write(src);
+ }
+
+ /**
+ * Write to the pipe
+ */
+ public int write(byte[] src) throws IOException {
+ checkFlipped();
+ return write(ByteBuffer.wrap(src));
+ }
+
+ /**
+ * Write to the pipe
+ */
+ public int write(byte[] src, int offset, int len) throws IOException {
+ checkFlipped();
+ return write(ByteBuffer.wrap(src, offset, len));
+ }
+
+ /**
+ * True if this pipe is readable
+ */
+ public boolean isReadable() {
+ return flipped;
+ }
+
+ /**
+ * True if this pipe is writable
+ */
+ public boolean isWritable() {
+ return !flipped;
+ }
+
+ /**
+ * If the pipe is writable, this will close the input and switch to readable mode
+ * If the pipe is readable, this will close the output and reset the pipe
+ */
+ public void close() throws IOException {
+ if (!flipped) {
+ if (pipe.sink().isOpen()) pipe.sink().close();
+ flipped = true;
+ } else {
+ if (pipe.source().isOpen()) pipe.source().close();
+ reset();
+ }
+ }
+
+ /**
+ * Reset the pipe. Switches the pipe to writable mode
+ */
+ public void reset() {
+ try {
+ if (pipe != null) {
+ if (pipe.sink().isOpen()) pipe.sink().close();
+ if (pipe.source().isOpen()) pipe.source().close();
+ }
+ pipe = Pipe.open();
+ flipped = false;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static class PipeChannelInputStream
+ extends FilterInputStream {
+ private final PipeChannel pipe;
+ protected PipeChannelInputStream(
+ PipeChannel pipe,
+ InputStream in) {
+ super(in);
+ this.pipe = pipe;
+ }
+ @Override
+ public void close() throws IOException {
+ pipe.close();
+ }
+ }
+
+ private static class PipeChannelOutputStream
+ extends FilterOutputStream {
+ private final PipeChannel pipe;
+ protected PipeChannelOutputStream(
+ PipeChannel pipe,
+ OutputStream in) {
+ super(in);
+ this.pipe = pipe;
+ }
+ @Override
+ public void close() throws IOException {
+ pipe.close();
+ }
+ }
+
+ private static class PipeChannelReader
+ extends FilterReader {
+ private final PipeChannel pipe;
+ protected PipeChannelReader(
+ PipeChannel pipe,
+ Reader in) {
+ super(in);
+ this.pipe = pipe;
+ }
+ @Override
+ public void close() throws IOException {
+ pipe.close();
+ }
+ }
+
+ private static class PipeChannelWriter
+ extends FilterWriter {
+ private final PipeChannel pipe;
+ protected PipeChannelWriter(
+ PipeChannel pipe,
+ Writer in) {
+ super(in);
+ this.pipe = pipe;
+ }
+ @Override
+ public void close() throws IOException {
+ pipe.close();
+ }
+ }
+
+ public Appendable append(
+ CharSequence csq)
+ throws IOException {
+ getWriter().append(csq);
+ return this;
+ }
+
+ public Appendable append(
+ char c)
+ throws IOException {
+ getWriter().append(c);
+ return this;
+ }
+
+ public Appendable append(
+ CharSequence csq,
+ int start,
+ int end)
+ throws IOException {
+ getWriter().append(csq,start,end);
+ return this;
+ }
+
+ public Appendable append(
+ CharSequence csq,
+ String charset)
+ throws IOException {
+ getWriter(charset).append(csq);
+ return this;
+ }
+
+ public Appendable append(
+ char c,
+ String charset)
+ throws IOException {
+ getWriter(charset).append(c);
+ return this;
+ }
+
+ public Appendable append(
+ CharSequence csq,
+ int start,
+ int end,
+ String charset)
+ throws IOException {
+ getWriter(charset).append(csq,start,end);
+ return this;
+ }
+
+ public int read(
+ CharBuffer cb)
+ throws IOException {
+ return getReader().read(cb);
+ }
+
+ public int read(
+ CharBuffer cb,
+ String charset)
+ throws IOException {
+ return getReader(charset).read(cb);
+ }
+}
Added: incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/RewindableInputStream.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/RewindableInputStream.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/RewindableInputStream.java (added)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/main/java/org/apache/abdera/i18n/text/io/RewindableInputStream.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,115 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+package org.apache.abdera.i18n.text.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * RewindableInputStream is a specialization of the PushbackInputStream
+ * that maintains an internal buffer of read bytes that a user can
+ * rewind (unread) back into the stream without having to do their
+ * own buffer management. The rewind buffer grows dynamically
+ */
+public class RewindableInputStream
+ extends DynamicPushbackInputStream {
+
+ private static final int INITIAL_CAPACITY = 32;
+ private byte[] buffer;
+ private int position;
+ private final int scale;
+
+ public RewindableInputStream(
+ InputStream in) {
+ this(in,INITIAL_CAPACITY);
+ }
+
+ public RewindableInputStream(
+ InputStream in,
+ int capacity) {
+ super(in);
+ grow(capacity);
+ this.scale = capacity;
+ }
+
+ public int position() {
+ return position;
+ }
+
+ private void grow(int capacity) {
+ if (buffer == null) {
+ buffer = new byte[capacity];
+ return;
+ } else {
+ byte[] buf = new byte[buffer.length + capacity];
+ System.arraycopy(buffer, 0, buf, 0, buffer.length);
+ buffer = buf;
+ }
+ }
+
+ private void shrink(int len) {
+ if (buffer == null) return;
+ byte[] buf = new byte[buffer.length-len];
+ System.arraycopy(buffer, 0, buf, 0, buf.length);
+ position = buffer.length-len;
+ buffer = buf;
+ }
+
+ public void rewind() throws IOException {
+ if (buffer.length == 0) return;
+ unread(buffer,0,position);
+ shrink(buffer.length);
+ }
+
+ public void rewind(int offset, int len) throws IOException {
+ if (buffer.length == 0) return;
+ if (offset > buffer.length)
+ throw new ArrayIndexOutOfBoundsException(offset);
+ unread(buffer,offset,len);
+ shrink(len);
+ }
+
+ public void rewind(int len) throws IOException {
+ if (buffer.length == 0) return;
+ rewind(buffer.length-len,len);
+ }
+
+ public int read() throws IOException {
+ int i = super.read();
+ if (i != -1) {
+ if (position >= buffer.length) grow(scale);
+ buffer[position++] = (byte) i;
+ }
+ return i;
+ }
+
+ public int read(byte[] b, int off, int len) throws IOException {
+ int r = super.read(b, off, len);
+ if (r != -1) {
+ if (position + r >= buffer.length) grow(Math.max(position+r,scale));
+ System.arraycopy(b, off, buffer, position, r);
+ position = position + r;
+ }
+ return r;
+ }
+
+ public long skip(long n) throws IOException {
+ return super.skip(n);
+ }
+
+}
Modified: incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestIRI.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestIRI.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestIRI.java (original)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestIRI.java Mon Dec 31 20:59:44 2007
@@ -22,7 +22,7 @@
import junit.framework.TestCase;
import org.apache.abdera.i18n.iri.IRI;
-import org.apache.abdera.i18n.unicode.Normalizer;
+import org.apache.abdera.i18n.text.Normalizer;
public class TestIRI extends TestCase {
Modified: incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNFKC.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNFKC.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNFKC.java (original)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNFKC.java Mon Dec 31 20:59:44 2007
@@ -17,7 +17,7 @@
*/
package org.apache.abdera.i18n.test.iri;
-import org.apache.abdera.i18n.unicode.Normalizer;
+import org.apache.abdera.i18n.text.Normalizer;
public class TestNFKC extends TestBase {
Modified: incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNameprep.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNameprep.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNameprep.java (original)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestNameprep.java Mon Dec 31 20:59:44 2007
@@ -17,8 +17,8 @@
*/
package org.apache.abdera.i18n.test.iri;
-import org.apache.abdera.i18n.io.CharUtils;
-import org.apache.abdera.i18n.iri.Nameprep;
+import org.apache.abdera.i18n.text.CharUtils;
+import org.apache.abdera.i18n.text.Nameprep;
public class TestNameprep extends TestBase {
@@ -68,38 +68,38 @@
string(0xE1,0xBD,0x96)),
N("ASCII space character U+0020", "\u0020" , "\u0020"),
O("Non-ASCII 8bit space character U+00A0", "\u00A0", ""),
- P("Non-ASCII multibyte space character U+1680", "\u1680", null, false, -1),
- Q("Non-ASCII multibyte space character U+2000", "\u2000", "\u0020", false, -1),
+ P("Non-ASCII multibyte space character U+1680", "\u1680", null, -1),
+ Q("Non-ASCII multibyte space character U+2000", "\u2000", "\u0020", -1),
R("Zero Width Space U+200b", "\u200B", ""),
- S("Non-ASCII multibyte space character U+3000", "\u3000", "\u0020",false,-1),
+ S("Non-ASCII multibyte space character U+3000", "\u3000", "\u0020",-1),
T("ASCII control characters U+0010 U+007F","\u0010\u007F","\u0010\u007F"),
- U("Non-ASCII 8bit control character U+0085", "\u0085", null, false, -1),
- V("Non-ASCII multibyte control character U+180E", "\u180E", null, false, -1),
+ U("Non-ASCII 8bit control character U+0085", "\u0085", null, -1),
+ V("Non-ASCII multibyte control character U+180E", "\u180E", null, -1),
W("Zero Width No-Break Space U+FEFF","\uFEFF",""),
X("Non-ASCII control character U+1D175",
new String(new char[] {
CharUtils.getHighSurrogate(0x1D175),
CharUtils.getLowSurrogate(0x1D175)}),
- null,false,-1),
- Y("Plane 0 private use character U+F123", "\uF123", null, false, -1),
- Z("Plane 15 private use character U+F1234", string(0xF3,0xB1,0x88,0xB4), null, false, -1),
- AA("Plane 16 private use character U+10F234", string(0xF4,0x8F,0x88,0xB4), null, false, -1),
- AB("Non-character code point U+8FFFE", string(0xF2,0x8F,0x8F,0xBE), null, false, -1),
- AC("Non-character code point U+10FFFF", string(0xF4,0x8F,0x8F,0x8F), null, false, -1),
- AD("Surrogate code U+DF42",string(0xED,0xBD,0x82),null,false,-1),
- AE("Non-plain text character U+FFFD", string(0xEF,0xBF,0xBD), null, false, -1),
- AF("Ideographic description character U+2FF5", string(0xE2,0xBF,0xB5), null, false, -1),
- AG("Display property character U+0341", string(0xCD,0x81), string(0xCC,0x81), false, -1),
- AH("Left-to-right mark U+200E",string(0xE2,0x80,0x8E),null,false, -1),
- AI("Deprecated U+202A", string(0xE2,0x80,0xAA), null, false, -1),
- AJ("Language tagging character U+E0001", string(0xF3,0xA0,0x80,0x81), null, false, -1),
- AK("Language tagging character U+E0042", string(0xF3,0xA0,0x81,0x82), null, false, -1),
- AL("Bidi: RandALCat character U+05BE and LCat characters", string('f','o','o',0xD6,0xBE), null, false, -1),
- AM("Bidi: RandALCat character U+FD50 and LCat characters", string('f','o','o',0xEF,0xB5,0x90), null, false, -1),
- AN("Bidi: RandALCat character U+FB38 and LCat characters", string('f','o','o',0xEF,0xB9,0xB6), null, false, -1),
- AO("Bidi: RandALCat without trailing RandALCat U+0627 U+0031", string(0xD8,0xA7,0x31), null, false, -1),
+ null,-1),
+ Y("Plane 0 private use character U+F123", "\uF123", null, -1),
+ Z("Plane 15 private use character U+F1234", string(0xF3,0xB1,0x88,0xB4), null, -1),
+ AA("Plane 16 private use character U+10F234", string(0xF4,0x8F,0x88,0xB4), null, -1),
+ AB("Non-character code point U+8FFFE", string(0xF2,0x8F,0x8F,0xBE), null, -1),
+ AC("Non-character code point U+10FFFF", string(0xF4,0x8F,0x8F,0x8F), null, -1),
+ AD("Surrogate code U+DF42",string(0xED,0xBD,0x82),null,-1),
+ AE("Non-plain text character U+FFFD", string(0xEF,0xBF,0xBD), null, -1),
+ AF("Ideographic description character U+2FF5", string(0xE2,0xBF,0xB5), null, -1),
+ AG("Display property character U+0341", string(0xCD,0x81), string(0xCC,0x81), -1),
+ AH("Left-to-right mark U+200E",string(0xE2,0x80,0x8E),null,-1),
+ AI("Deprecated U+202A", string(0xE2,0x80,0xAA), null, -1),
+ AJ("Language tagging character U+E0001", string(0xF3,0xA0,0x80,0x81), null, -1),
+ AK("Language tagging character U+E0042", string(0xF3,0xA0,0x81,0x82), null, -1),
+ AL("Bidi: RandALCat character U+05BE and LCat characters", string('f','o','o',0xD6,0xBE), null, -1),
+ AM("Bidi: RandALCat character U+FD50 and LCat characters", string('f','o','o',0xEF,0xB5,0x90), null, -1),
+ AN("Bidi: RandALCat character U+FB38 and LCat characters", string('f','o','o',0xEF,0xB9,0xB6), null, -1),
+ AO("Bidi: RandALCat without trailing RandALCat U+0627 U+0031", string(0xD8,0xA7,0x31), null, -1),
AP("Bidi: RandALCat character U+0627 U+0031 U+0628", string(0xD8,0xA7,0x31,0xD8,0xA8), string(0xD8,0xA7,0x31,0xD8,0xA8)),
- AQ("Unassigned code point U+E0002",string(0xF3,0xA0,0x80,0x82),null,true,-1),
+ //AQ("Unassigned code point U+E0002",string(0xF3,0xA0,0x80,0x82),null,true,-1),
// AR("Larger test (shrinking)", // {"Larger test (shrinking)","X\xC2\xAD\xC3\x9F\xC4\xB0\xE2\x84\xA1\x6a\xcc\x8c\xc2\xa0\xc2""\xaa\xce\xb0\xe2\x80\x80", "xssi\xcc\x87" "tel\xc7\xb0 a\xce\xb0 ","Nameprep"},
// string('X',0xC2,0xAD,0xC3,0x9F,0xC4,0xB0,0xE2,0x84,0xA1,0x6a,0xc,'c',0x8c,0xc2,0xa0,0xc2),
// string('x','s','s','i',0xcc,0x87,'t','e','l',0xc7,0xb0,'a',0xce,0xb0), false, 0),
@@ -110,17 +110,16 @@
AU("NFKC test", string(0xC2,0xAA), string(0x61)),
AV("nameprep, exposed a bug in libstringprep 0.0.5",
string(0xC2,0xAA,0x0A), string(0x61,0x0A)),
- AW("unassigned code point U+0221", "\u0221", "\u0221", true, 0),
- AX("unassigned code point U+0221", "\u0221", "\u0221", false, -1),
- AY("Unassigned code point U+0236", "\u0236", "\u0236", true, 0),
- AZ("unassigned code point U+0236", "\u0236", "\u0236", false, -1),
+ // AW("unassigned code point U+0221", "\u0221", "\u0221", true, 0),
+ AX("unassigned code point U+0221", "\u0221", "\u0221", -1),
+ //AY("Unassigned code point U+0236", "\u0236", "\u0236", true, 0),
+ AZ("unassigned code point U+0236", "\u0236", "\u0236", -1),
BA("bidi both RandALCat and LCat U+0627 U+00AA U+0628",
- string(0xD8,0xA7,0xC2,0xAA,0xD8,0xA8), null, false, -1);
+ string(0xD8,0xA7,0xC2,0xAA,0xD8,0xA8), null, -1);
String comment;
String in;
String out;
- boolean allowunassigned = true;
int rc;
Test(
@@ -136,12 +135,10 @@
String comment,
String in,
String out,
- boolean allowunassigned,
int rc) {
this.comment = comment;
this.in = in;
this.out = out;
- this.allowunassigned = allowunassigned;
this.rc = rc;
}
@@ -150,7 +147,7 @@
public static void testNameprep() throws Exception {
for (Test test : Test.values()) {
try {
- String out = Nameprep.prep(test.in,test.allowunassigned);
+ String out = Nameprep.prep(test.in);
assertEquals(test.out,out);
} catch (Exception e) {
if (test.rc != -1)
Modified: incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestPunycode.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestPunycode.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestPunycode.java (original)
+++ incubator/abdera/java/trunk/dependencies/i18n/src/test/java/org/apache/abdera/i18n/test/iri/TestPunycode.java Mon Dec 31 20:59:44 2007
@@ -17,7 +17,7 @@
*/
package org.apache.abdera.i18n.test.iri;
-import org.apache.abdera.i18n.iri.Punycode;
+import org.apache.abdera.i18n.text.Punycode;
public class TestPunycode extends TestBase {
Added: incubator/abdera/java/trunk/examples/src/main/java/org/apache/abdera/examples/unicode/NormalizationExample.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/examples/src/main/java/org/apache/abdera/examples/unicode/NormalizationExample.java?rev=607801&view=auto
==============================================================================
--- incubator/abdera/java/trunk/examples/src/main/java/org/apache/abdera/examples/unicode/NormalizationExample.java (added)
+++ incubator/abdera/java/trunk/examples/src/main/java/org/apache/abdera/examples/unicode/NormalizationExample.java Mon Dec 31 20:59:44 2007
@@ -0,0 +1,52 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. The ASF licenses this file to You
+* under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License. For additional information regarding
+* copyright in this work, please see the NOTICE file in the top level
+* directory of this distribution.
+*/
+package org.apache.abdera.examples.unicode;
+
+import org.apache.abdera.i18n.text.Normalizer;
+
+/**
+ * Example that demonstrates Abdera's basic support for Unicode normalization.
+ */
+public class NormalizationExample {
+
+ public static void main(String...args) throws Exception {
+
+ // Three different representations of the same character (Angstrom)
+ String s1 = "\u00c5";
+ String s2 = "\u0041\u030A";
+ String s3 = "\u212B";
+
+ System.out.println(s1 + "=" + s2 + " ?\t" + s1.equals(s2)); // false
+ System.out.println(s1 + "=" + s3 + " ?\t" + s1.equals(s3)); // false
+ System.out.println(s2 + "=" + s3 + " ?\t" + s2.equals(s3)); // false
+
+ // Normalize to NFC
+ String n1 = Normalizer.normalize(s1, Normalizer.Form.C);
+ String n2 = Normalizer.normalize(s2, Normalizer.Form.C);
+ String n3 = Normalizer.normalize(s3, Normalizer.Form.C);
+
+ System.out.println(n1 + "=" + n2 + " ?\t" + n1.equals(n2)); // true
+ System.out.println(n1 + "=" + n3 + " ?\t" + n1.equals(n3)); // true
+ System.out.println(n2 + "=" + n3 + " ?\t" + n2.equals(n3)); // true
+
+ // s1 is already normalized to NFC
+ System.out.println(n1.equals(s1)); // true
+
+ }
+
+}
Modified: incubator/abdera/java/trunk/extensions/converters/src/main/java/org/apache/abdera/converter/impl/ContentConverter.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/extensions/converters/src/main/java/org/apache/abdera/converter/impl/ContentConverter.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/extensions/converters/src/main/java/org/apache/abdera/converter/impl/ContentConverter.java (original)
+++ incubator/abdera/java/trunk/extensions/converters/src/main/java/org/apache/abdera/converter/impl/ContentConverter.java Mon Dec 31 20:59:44 2007
@@ -36,8 +36,8 @@
import org.apache.abdera.converter.annotation.ContentType;
import org.apache.abdera.converter.annotation.MediaType;
import org.apache.abdera.converter.annotation.Value;
-import org.apache.abdera.i18n.io.InputStreamDataSource;
import org.apache.abdera.i18n.iri.IRI;
+import org.apache.abdera.i18n.text.io.InputStreamDataSource;
import org.apache.abdera.model.Content;
import org.apache.abdera.model.Div;
import org.apache.abdera.model.Document;
Modified: incubator/abdera/java/trunk/extensions/json/src/main/java/org/apache/abdera/ext/json/JSONUtil.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/extensions/json/src/main/java/org/apache/abdera/ext/json/JSONUtil.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/extensions/json/src/main/java/org/apache/abdera/ext/json/JSONUtil.java (original)
+++ incubator/abdera/java/trunk/extensions/json/src/main/java/org/apache/abdera/ext/json/JSONUtil.java Mon Dec 31 20:59:44 2007
@@ -28,9 +28,9 @@
import org.apache.abdera.ext.html.HtmlHelper;
import org.apache.abdera.ext.thread.InReplyTo;
import org.apache.abdera.ext.thread.ThreadHelper;
-import org.apache.abdera.i18n.io.CharUtils.Profile;
-import org.apache.abdera.i18n.iri.Escaping;
import org.apache.abdera.i18n.iri.IRI;
+import org.apache.abdera.i18n.text.UrlEncoding;
+import org.apache.abdera.i18n.text.CharUtils.Profile;
import org.apache.abdera.model.Base;
import org.apache.abdera.model.Categories;
import org.apache.abdera.model.Category;
@@ -396,7 +396,7 @@
"src".equalsIgnoreCase(name) ||
"action".equalsIgnoreCase(name))) {
IRI base = child.getResolvedBaseUri();
- val = Escaping.encode(val.trim(),Profile.IUNRESERVED,Profile.RESERVED,Profile.IPRIVATE);
+ val = UrlEncoding.encode(val.trim(),Profile.IUNRESERVED.filter(),Profile.RESERVED.filter(),Profile.IPRIVATE.filter());
if (base != null) val = base.resolve(val).toASCIIString();
}
jstream.writeQuoted(val);
Modified: incubator/abdera/java/trunk/extensions/main/src/main/java/org/apache/abdera/ext/bidi/BidiHelper.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/extensions/main/src/main/java/org/apache/abdera/ext/bidi/BidiHelper.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/extensions/main/src/main/java/org/apache/abdera/ext/bidi/BidiHelper.java (original)
+++ incubator/abdera/java/trunk/extensions/main/src/main/java/org/apache/abdera/ext/bidi/BidiHelper.java Mon Dec 31 20:59:44 2007
@@ -24,8 +24,8 @@
import javax.xml.namespace.QName;
-import org.apache.abdera.i18n.io.CharUtils;
import org.apache.abdera.i18n.lang.Lang;
+import org.apache.abdera.i18n.text.CharUtils;
import org.apache.abdera.model.Base;
import org.apache.abdera.model.Document;
import org.apache.abdera.model.Element;
@@ -118,8 +118,8 @@
*/
public static String getBidiText(Direction direction, String text) {
switch (direction) {
- case LTR: return CharUtils.bidiLRE(text);
- case RTL: return CharUtils.bidiRLE(text);
+ case LTR: return CharUtils.wrapBidi(text,CharUtils.LRE);
+ case RTL: return CharUtils.wrapBidi(text,CharUtils.RLE);
default: return text;
}
}
Modified: incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/FOMEntry.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/FOMEntry.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/FOMEntry.java (original)
+++ incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/FOMEntry.java Mon Dec 31 20:59:44 2007
@@ -45,7 +45,7 @@
import org.apache.abdera.parser.stax.util.FOMHelper;
import org.apache.abdera.util.Constants;
import org.apache.abdera.util.URIHelper;
-import org.apache.abdera.i18n.io.InputStreamDataSource;
+import org.apache.abdera.i18n.text.io.InputStreamDataSource;
import org.apache.abdera.i18n.iri.IRI;
import org.apache.axiom.om.OMContainer;
import org.apache.axiom.om.OMElement;
Modified: incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMSniffingInputStream.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMSniffingInputStream.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMSniffingInputStream.java (original)
+++ incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMSniffingInputStream.java Mon Dec 31 20:59:44 2007
@@ -23,8 +23,8 @@
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamReader;
-import org.apache.abdera.i18n.io.CharsetSniffingInputStream;
-import org.apache.abdera.i18n.io.PeekAheadInputStream;
+import org.apache.abdera.i18n.text.io.CharsetSniffingInputStream;
+import org.apache.abdera.i18n.text.io.PeekAheadInputStream;
/**
* Will attempt to autodetect the character encoding from the stream
Modified: incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMXmlVersionInputStream.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMXmlVersionInputStream.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMXmlVersionInputStream.java (original)
+++ incubator/abdera/java/trunk/parser/src/main/java/org/apache/abdera/parser/stax/util/FOMXmlVersionInputStream.java Mon Dec 31 20:59:44 2007
@@ -23,7 +23,8 @@
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamReader;
-import org.apache.abdera.i18n.io.PeekAheadInputStream;
+
+import org.apache.abdera.i18n.text.io.PeekAheadInputStream;
/**
* Will attempt to autodetect the character encoding from the stream
Modified: incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/EncodingTest.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/EncodingTest.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/EncodingTest.java (original)
+++ incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/EncodingTest.java Mon Dec 31 20:59:44 2007
@@ -23,9 +23,12 @@
import java.io.StringReader;
import java.util.Date;
+import junit.framework.TestCase;
+
import org.apache.abdera.Abdera;
import org.apache.abdera.model.Content;
import org.apache.abdera.model.Document;
+import org.apache.abdera.model.Element;
import org.apache.abdera.model.Entry;
import org.apache.abdera.parser.Parser;
import org.apache.abdera.parser.ParserOptions;
@@ -33,8 +36,6 @@
import org.apache.abdera.util.CompressionUtil.CompressionCodec;
import org.apache.abdera.writer.WriterOptions;
-import junit.framework.TestCase;
-
public class EncodingTest extends TestCase {
@@ -81,7 +82,7 @@
Parser parser = abdera.getParser();
ParserOptions options = parser.getDefaultParserOptions();
options.setFilterRestrictedCharacters(true);
- Document doc = parser.parse(new StringReader(s), null, options);
+ Document<Element> doc = parser.parse(new StringReader(s), null, options);
doc.getRoot().toString();
}
@@ -95,7 +96,7 @@
ParserOptions options = parser.getDefaultParserOptions();
options.setFilterRestrictedCharacters(true);
options.setCharset("UTF-8");
- Document doc = parser.parse(new ByteArrayInputStream(s.getBytes("UTF-8")), null, options);
+ Document<Element> doc = parser.parse(new ByteArrayInputStream(s.getBytes("UTF-8")), null, options);
doc.getRoot().toString();
}
Modified: incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/stax/FeedParserTest.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/stax/FeedParserTest.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/stax/FeedParserTest.java (original)
+++ incubator/abdera/java/trunk/parser/src/test/java/org/apache/abdera/test/parser/stax/FeedParserTest.java Mon Dec 31 20:59:44 2007
@@ -23,12 +23,12 @@
import javax.activation.DataHandler;
+import org.apache.abdera.i18n.iri.IRI;
import org.apache.abdera.model.Content;
import org.apache.abdera.model.Document;
import org.apache.abdera.model.Entry;
import org.apache.abdera.model.Feed;
import org.apache.abdera.model.Person;
-import org.apache.abdera.i18n.iri.IRI;
public class FeedParserTest extends BaseParserTestCase {
@@ -44,8 +44,8 @@
}
public void testEntryAuthorEmail() throws Exception {
- Document doc = parse(baseURI.resolve("entry_author_email.xml"));
- Feed feed = (Feed) doc.getRoot();
+ Document<Feed> doc = parse(baseURI.resolve("entry_author_email.xml"));
+ Feed feed = doc.getRoot();
Entry entry = feed.getEntries().get(0);
Person person = entry.getAuthor();
assertEquals(person.getEmail(), "me@example.com");
Modified: incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/AbstractMessage.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/AbstractMessage.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/AbstractMessage.java (original)
+++ incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/AbstractMessage.java Mon Dec 31 20:59:44 2007
@@ -19,8 +19,8 @@
import javax.activation.MimeType;
-import org.apache.abdera.i18n.iri.Escaping;
import org.apache.abdera.i18n.iri.IRI;
+import org.apache.abdera.i18n.text.UrlEncoding;
import org.apache.abdera.protocol.Message;
/**
@@ -58,13 +58,13 @@
}
public String getDecodedHeader(String header) {
- return Escaping.decode(EncodingUtil.decode(getHeader(header)));
+ return UrlEncoding.decode(EncodingUtil.decode(getHeader(header)));
}
public String[] getDecodedHeaders(String header) {
Object[] headers = getHeaders(header);
for (int n = 0; n < headers.length; n++) {
- headers[n] = Escaping.decode(EncodingUtil.decode(headers[n].toString()));
+ headers[n] = UrlEncoding.decode(EncodingUtil.decode(headers[n].toString()));
}
return (String[])headers;
}
Modified: incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/EncodingUtil.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/EncodingUtil.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/EncodingUtil.java (original)
+++ incubator/abdera/java/trunk/protocol/src/main/java/org/apache/abdera/protocol/util/EncodingUtil.java Mon Dec 31 20:59:44 2007
@@ -20,9 +20,8 @@
import java.io.IOException;
import java.io.InputStream;
-import org.apache.abdera.i18n.io.CharUtils.Profile;
-import org.apache.abdera.i18n.iri.Escaping;
-import org.apache.abdera.i18n.unicode.Normalizer;
+import org.apache.abdera.i18n.text.Normalizer;
+import org.apache.abdera.i18n.text.Sanitizer;
import org.apache.abdera.util.CompressionUtil;
import org.apache.abdera.util.CompressionUtil.CompressionCodec;
import org.apache.commons.codec.DecoderException;
@@ -33,32 +32,56 @@
public static final String SANITIZE_PATTERN = "[^A-Za-z0-9\\%!$&\\\\'()*+,;=]+";
+ /**
+ * @deprecated
+ * @see org.apache.abdera.i18n.text.Sanitizer
+ */
public static String sanitize(String slug) {
- return sanitize(slug, null, false, null, SANITIZE_PATTERN);
+ return Sanitizer.sanitize(slug, null, false, null, SANITIZE_PATTERN);
}
+ /**
+ * @deprecated
+ * @see org.apache.abdera.i18n.text.Sanitizer
+ */
public static String sanitize(String slug, String filler) {
- return sanitize(slug, filler, false, null, SANITIZE_PATTERN);
+ return Sanitizer.sanitize(slug, filler, false, null, SANITIZE_PATTERN);
}
+ /**
+ * @deprecated
+ * @see org.apache.abdera.i18n.text.Sanitizer
+ */
public static String sanitize(String slug, String filler, boolean lower) {
- return sanitize(slug, filler, lower, null, SANITIZE_PATTERN);
+ return Sanitizer.sanitize(slug, filler, lower, null, SANITIZE_PATTERN);
}
+ /**
+ * @deprecated
+ * @see org.apache.abdera.i18n.text.Sanitizer
+ */
public static String sanitize(String slug, String filler, String pattern) {
- return sanitize(slug, filler, false, null, pattern);
+ return Sanitizer.sanitize(slug, filler, false, null, pattern);
}
+ /**
+ * @deprecated
+ * @see org.apache.abdera.i18n.text.Sanitizer
+ */
public static String sanitize(String slug, String filler, boolean lower, String pattern) {
- return sanitize(slug, filler, lower, null, pattern);
+ return Sanitizer.sanitize(slug, filler, lower, null, pattern);
}
+ /**
+ * @deprecated
+ * @see org.apache.abdera.i18n.text.Sanitizer
+ */
public static String sanitize(
String slug,
String filler,
boolean lower,
Normalizer.Form form) {
- return sanitize(slug,filler,lower,form,SANITIZE_PATTERN);
+ return Sanitizer.sanitize(slug,filler,lower,form,SANITIZE_PATTERN);
}
/**
@@ -69,6 +92,8 @@
* @param filler The replacement string
* @param lower True if the result should be lowercase
* @param form Unicode Normalization form to use (or null)
+ * @deprecated
+ * @see org.apache.abdera.i18n.text.Sanitizer
*/
public static String sanitize(
String slug,
@@ -76,21 +101,7 @@
boolean lower,
Normalizer.Form form,
String pattern) {
- if (slug == null) return null;
- if (lower) slug = slug.toLowerCase();
- if (form != null) {
- try {
- slug =
- Normalizer.normalize(
- slug, form);
- } catch (Exception e) {}
- }
- if (filler != null) {
- slug = slug.replaceAll(pattern,filler);
- } else {
- slug = Escaping.encode(slug, Profile.PATHNODELIMS);
- }
- return slug;
+ return Sanitizer.sanitize(slug,filler,lower,form,pattern);
}
public static enum Codec { B, Q };
Modified: incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/ResponseContext.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/ResponseContext.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/ResponseContext.java (original)
+++ incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/ResponseContext.java Mon Dec 31 20:59:44 2007
@@ -21,7 +21,7 @@
import java.io.OutputStream;
import java.util.Date;
-import org.apache.abdera.i18n.io.CharUtils.Profile;
+import org.apache.abdera.i18n.text.CharUtils.Profile;
import org.apache.abdera.protocol.Response;
import org.apache.abdera.util.EntityTag;
import org.apache.abdera.writer.Writer;
Modified: incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractCollectionProvider.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractCollectionProvider.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractCollectionProvider.java (original)
+++ incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractCollectionProvider.java Mon Dec 31 20:59:44 2007
@@ -26,8 +26,8 @@
import org.apache.abdera.Abdera;
import org.apache.abdera.factory.Factory;
-import org.apache.abdera.i18n.iri.Escaping;
import org.apache.abdera.i18n.iri.IRI;
+import org.apache.abdera.i18n.text.UrlEncoding;
import org.apache.abdera.model.Base;
import org.apache.abdera.model.Content;
import org.apache.abdera.model.Document;
@@ -514,7 +514,7 @@
}
String[] segments = path.split("/");
String id = segments[segments.length - 1];
- return Escaping.decode(id);
+ return UrlEncoding.decode(id);
}
protected IRI getMediaIRI(IRI entryBaseIri, String name) {
Modified: incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractResponseContext.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractResponseContext.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractResponseContext.java (original)
+++ incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractResponseContext.java Mon Dec 31 20:59:44 2007
@@ -26,8 +26,8 @@
import javax.activation.MimeType;
-import org.apache.abdera.i18n.io.CharUtils.Profile;
-import org.apache.abdera.i18n.iri.Escaping;
+import org.apache.abdera.i18n.text.UrlEncoding;
+import org.apache.abdera.i18n.text.CharUtils.Profile;
import org.apache.abdera.protocol.server.ResponseContext;
import org.apache.abdera.protocol.util.AbstractResponse;
import org.apache.abdera.protocol.util.EncodingUtil;
@@ -66,7 +66,7 @@
}
public ResponseContext setEscapedHeader(String name, Profile profile, String value) {
- return setHeader(name,Escaping.encode(value, profile));
+ return setHeader(name,UrlEncoding.encode(value, profile.filter()));
}
public ResponseContext setHeader(String name, Object value) {
Modified: incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractServiceProvider.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractServiceProvider.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractServiceProvider.java (original)
+++ incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/impl/AbstractServiceProvider.java Mon Dec 31 20:59:44 2007
@@ -23,14 +23,9 @@
import java.util.Set;
import org.apache.abdera.Abdera;
-import org.apache.abdera.factory.Factory;
-import org.apache.abdera.i18n.io.CharUtils.Profile;
-import org.apache.abdera.i18n.iri.Escaping;
import org.apache.abdera.i18n.iri.IRI;
-import org.apache.abdera.model.Collection;
-import org.apache.abdera.model.Document;
-import org.apache.abdera.model.Service;
-import org.apache.abdera.model.Workspace;
+import org.apache.abdera.i18n.text.UrlEncoding;
+import org.apache.abdera.i18n.text.CharUtils.Profile;
import org.apache.abdera.protocol.Request;
import org.apache.abdera.protocol.Resolver;
import org.apache.abdera.protocol.server.CollectionProvider;
@@ -79,7 +74,7 @@
path = path.substring(0, q);
}
- path = Escaping.decode(path);
+ path = UrlEncoding.decode(path);
CollectionProvider provider = null;
String providerHref = null;
@@ -161,7 +156,7 @@
String href;
try {
- href = Escaping.encode(entry.getKey(), enc, Profile.PATH);
+ href = UrlEncoding.encode(entry.getKey(), enc, Profile.PATH.filter());
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
Modified: incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/servlet/AbstractFilter.java
URL: http://svn.apache.org/viewvc/incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/servlet/AbstractFilter.java?rev=607801&r1=607800&r2=607801&view=diff
==============================================================================
--- incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/servlet/AbstractFilter.java (original)
+++ incubator/abdera/java/trunk/server/src/main/java/org/apache/abdera/protocol/server/servlet/AbstractFilter.java Mon Dec 31 20:59:44 2007
@@ -39,7 +39,7 @@
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpServletResponseWrapper;
-import org.apache.abdera.i18n.io.RewindableInputStream;
+import org.apache.abdera.i18n.text.io.RewindableInputStream;
import org.apache.abdera.util.CompressionUtil;
import org.apache.abdera.util.CompressionUtil.CompressionCodec;