You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by rv...@apache.org on 2013/11/12 16:53:37 UTC
svn commit: r1541118 - in
/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot:
lang/BlankNodeAllocatorFixedSeedHash.java lang/BlankNodeAllocatorHash.java
lang/LabelToNode.java tokens/TokenizerFactory.java
Author: rvesse
Date: Tue Nov 12 15:53:36 2013
New Revision: 1541118
URL: http://svn.apache.org/r1541118
Log:
Add some additional useful overloads and a blank node allocation policy with guaranteably reproducible hash based allocations
Added:
jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorFixedSeedHash.java
Modified:
jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorHash.java
jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/LabelToNode.java
jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java
Added: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorFixedSeedHash.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorFixedSeedHash.java?rev=1541118&view=auto
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorFixedSeedHash.java (added)
+++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorFixedSeedHash.java Tue Nov 12 15:53:36 2013
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.riot.lang;
+
+import java.util.UUID;
+
+/**
+ * A variant of {@link BlankNodeAllocatorHash} where a fixed seed is used so
+ * repeated runs produce identical allocations
+ *
+ */
+public class BlankNodeAllocatorFixedSeedHash extends BlankNodeAllocatorHash {
+
+ private UUID seed;
+
+ /**
+ * Creates a new allocator which will use a new random seed but that seed
+ * will remain fixed
+ */
+ public BlankNodeAllocatorFixedSeedHash() {
+ this(UUID.randomUUID());
+ }
+
+ /**
+ * Creates a new allocator which will use a fixed seed
+ *
+ * @param seed
+ * Seed
+ */
+ public BlankNodeAllocatorFixedSeedHash(UUID seed) {
+ super();
+ if (seed == null)
+ throw new NullPointerException("seed cannot be null");
+ this.seed = seed;
+ }
+
+ protected UUID freshSeed() {
+ return this.seed;
+ }
+}
Modified: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorHash.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorHash.java?rev=1541118&r1=1541117&r2=1541118&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorHash.java (original)
+++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/BlankNodeAllocatorHash.java Tue Nov 12 15:53:36 2013
@@ -38,12 +38,12 @@ import com.hp.hpl.jena.rdf.model.AnonId
* This is the most scalable, always legal allocator.
* <p>
* New allocators must be created per parser run, or .reset() called. These are
- * fed to a digest to gve a bit string, (currently MD5, to get a 128bit bit
+ * fed to a digest to give a bit string, (currently MD5, to get a 128bit bit
* string) that is used to form a bNode AnonId of hex digits.
* <p>
* In addition there is a cache of label->node allocations, using the natural
- * tendendency to locality in a database dump. (subject bNodes, bNodes in lists
- * and other datavalues structures like unit values).
+ * tendency to locality in a database dump. (subject bNodes, bNodes in lists
+ * and other data values structures like unit values).
* <p>
* Not thread safe.
*/
@@ -75,10 +75,18 @@ public class BlankNodeAllocatorHash impl
Cache<String, Node> cache1 = CacheFactory.createCache(CacheSize) ;
cache = CacheFactory.createCacheWithGetter(cache1, getter) ;
}
+
+ /**
+ * Gets a fresh seed value
+ * @return Seed value
+ */
+ protected UUID freshSeed() {
+ return UUID.randomUUID();
+ }
@Override
public void reset() {
- UUID seed = UUID.randomUUID() ;
+ UUID seed = this.freshSeed();
seedBytes = new byte[128 / 8] ;
Bytes.setLong(seed.getMostSignificantBits(), seedBytes, 0) ;
Bytes.setLong(seed.getLeastSignificantBits(), seedBytes, 8) ;
Modified: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/LabelToNode.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/LabelToNode.java?rev=1541118&r1=1541117&r2=1541118&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/LabelToNode.java (original)
+++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/lang/LabelToNode.java Tue Nov 12 15:53:36 2013
@@ -19,6 +19,7 @@
package org.apache.jena.riot.lang;
import java.util.HashMap ;
import java.util.Map ;
+import java.util.UUID;
import org.apache.jena.riot.out.NodeFmtLib ;
import org.apache.jena.riot.out.NodeToLabel ;
@@ -41,8 +42,15 @@ public class LabelToNode extends MapWith
*/
public static LabelToNode createScopeByDocumentHash()
{ return new LabelToNode(new AllocScopePolicy(), nodeAllocatorHash()) ; }
+
+ /**
+ * Allocation from a single scope; just the label matters. Use this policy if repeated runs must give identical allocations
+ * @param seed Seed
+ */
+ public static LabelToNode createScopeByDocumentHash(UUID seed)
+ { return new LabelToNode(new AllocScopePolicy(), nodeAllocatorHash(seed)) ; }
- /** The policy upto jena 2.10.0 - problems at very large scale */
+ /** The policy up to jena 2.10.0 - problems at very large scale */
public static LabelToNode createScopeByDocumentOld()
{ return new LabelToNode(new SingleScopePolicy(), nodeAllocatorTraditional()) ; }
@@ -86,6 +94,10 @@ public class LabelToNode extends MapWith
return new Alloc(new BlankNodeAllocatorHash()) ;
}
+ private static Allocator<String, Node, Node> nodeAllocatorHash(UUID seed) {
+ return new Alloc(new BlankNodeAllocatorFixedSeedHash(seed)) ;
+ }
+
private static Allocator<String, Node, Node> nodeAllocatorDeterministic() {
return new Alloc(new BlankNodeAllocatorLabel()) ;
}
Modified: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java
URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java?rev=1541118&r1=1541117&r2=1541118&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java (original)
+++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java Tue Nov 12 15:53:36 2013
@@ -42,6 +42,13 @@ public class TokenizerFactory
Tokenizer tokenizer = new TokenizerText(peekReader) ;
return tokenizer ;
}
+
+ public static Tokenizer makeTokenizerUTF8(String string)
+ {
+ PeekReader peekReader = PeekReader.readString(string);
+ Tokenizer tokenizer = new TokenizerText(peekReader);
+ return tokenizer;
+ }
public static Tokenizer makeTokenizerASCII(InputStream in)
{
Re: svn commit: r1541118 - in
/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot:
lang/BlankNodeAllocatorFixedSeedHash.java lang/BlankNodeAllocatorHash.java
lang/LabelToNode.java tokens/TokenizerFactory.java
Posted by Rob Vesse <rv...@dotnetrdf.org>.
Gah - character encoding in Java is still a horrific minefield :(
Removed the unnecessary overload
Rob
On 12/11/2013 18:55, "Andy Seaborne" <an...@apache.org> wrote:
>>
>>jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFa
>>ctory.java
>>
>
>
>> Modified:
>>jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFa
>>ctory.java
>> URL:
>>http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache
>>/jena/riot/tokens/TokenizerFactory.java?rev=1541118&r1=1541117&r2=1541118
>>&view=diff
>>
>>=========================================================================
>>=====
>> ---
>>jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFa
>>ctory.java (original)
>> +++
>>jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFa
>>ctory.java Tue Nov 12 15:53:36 2013
>> @@ -42,6 +42,13 @@ public class TokenizerFactory
>> Tokenizer tokenizer = new TokenizerText(peekReader) ;
>> return tokenizer ;
>> }
>> +
>> + public static Tokenizer makeTokenizerUTF8(String string)
>> + {
>> + PeekReader peekReader = PeekReader.readString(string);
>> + Tokenizer tokenizer = new TokenizerText(peekReader);
>> + return tokenizer;
>> + }
>>
>> public static Tokenizer makeTokenizerASCII(InputStream in)
>> {
>>
>>
>
>Rob -
>
>There is TokenizerFactory.makeTokenizerString which is identical to
>makeTokenizerUTF8. "String" was a better name because a string isn't
>UTF8 in Java.
>
> Andy
>
Re: svn commit: r1541118 - in /jena/trunk/jena-arq/src/main/java/org/apache/jena/riot:
lang/BlankNodeAllocatorFixedSeedHash.java lang/BlankNodeAllocatorHash.java
lang/LabelToNode.java tokens/TokenizerFactory.java
Posted by Andy Seaborne <an...@apache.org>.
> jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java
>
> Modified: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java
> URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java?rev=1541118&r1=1541117&r2=1541118&view=diff
> ==============================================================================
> --- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java (original)
> +++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerFactory.java Tue Nov 12 15:53:36 2013
> @@ -42,6 +42,13 @@ public class TokenizerFactory
> Tokenizer tokenizer = new TokenizerText(peekReader) ;
> return tokenizer ;
> }
> +
> + public static Tokenizer makeTokenizerUTF8(String string)
> + {
> + PeekReader peekReader = PeekReader.readString(string);
> + Tokenizer tokenizer = new TokenizerText(peekReader);
> + return tokenizer;
> + }
>
> public static Tokenizer makeTokenizerASCII(InputStream in)
> {
>
>
Rob -
There is TokenizerFactory.makeTokenizerString which is identical to
makeTokenizerUTF8. "String" was a better name because a string isn't
UTF8 in Java.
Andy