You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2017/09/28 16:05:46 UTC
[034/100] [abbrv] [partial] jena git commit: JENA-1397: Rename
modules and set versions
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/Record.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/Record.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/Record.java
new file mode 100644
index 0000000..f06d383
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/Record.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.record;
+
+import static java.lang.String.format ;
+
+import java.util.Arrays ;
+
+import org.apache.jena.atlas.lib.Bytes ;
+import org.seaborne.dboe.sys.SystemIndex ;
+
+/** A record is pair of key and value. It may be all key, in which case value is null. */
+
+final
+public class Record //implements Comparable<Record>
+{
+ /*
+ * Records of fixed size (controlled by the factory).
+ */
+ public static final Record NO_REC = null ;
+
+ final private byte[] key ;
+ final private byte[] value ;
+
+ public Record(byte[] key, byte[] value)
+ {
+ this.key = key ;
+ this.value = value ;
+ if ( SystemIndex.Checking )
+ {
+ if ( value != null && value.length == 0 )
+ throw new RecordException("Zero length value") ;
+ }
+ }
+
+ public byte[] getKey() { return key ; }
+ public byte[] getValue() { return value ; }
+
+ public boolean hasSeparateValue() { return value!=null ; }
+
+ @Override
+ public int hashCode()
+ {
+ return Arrays.hashCode(key) ^ Arrays.hashCode(value) ;
+ }
+
+ @Override
+ public boolean equals(Object other)
+ {
+ if ( this == other ) return true ;
+ if ( ! ( other instanceof Record ) ) return false ;
+ Record r = (Record)other ;
+ return compareByKeyValue(this, r) == 0 ;
+ }
+
+ @Override
+ public String toString()
+ {
+ if ( value == null )
+ return str(key) ;
+ return str(key)+":"+str(value) ;
+ }
+
+ /** Is the key of record1 == key of record2 */
+ public static boolean keyEQ(Record record1, Record record2)
+ {
+ int x = compareByKey(record1, record2) ;
+ return x == 0 ;
+ }
+
+ /** Is the key of record1 != key of record2 */
+ public static boolean keyNE(Record record1, Record record2)
+ {
+ int x = compareByKey(record1, record2) ;
+ return x != 0 ;
+ }
+
+ /** Is the key of record1 < key of record2 */
+ public static boolean keyLT(Record record1, Record record2)
+ {
+ int x = compareByKey(record1, record2) ;
+ return x < 0 ;
+ }
+
+ /** Is the key of record1 <= key of record2 */
+ public static boolean keyLE(Record record1, Record record2)
+ {
+ int x = compareByKey(record1, record2) ;
+ return x <= 0 ;
+ }
+
+ /** Is the key of record1 >= key of record2 */
+ public static boolean keyGE(Record record1, Record record2)
+ {
+ int x = compareByKey(record1, record2) ;
+ return x >= 0 ;
+ }
+
+ /** Is the key of record1 > key of record2 */
+ public static boolean keyGT(Record record1, Record record2)
+ {
+ int x = compareByKey(record1, record2) ;
+ return x > 0 ;
+ }
+
+ /** Is (key, value) of record1 equal to (key,value) of record2 */
+ public static boolean equals(Record record1, Record record2)
+ {
+ int x = compareByKeyValue(record1, record2) ;
+ return x == 0 ;
+ }
+
+
+
+ static public String str(byte[] b)
+ {
+ StringBuilder str = new StringBuilder() ;
+ for ( byte aB : b )
+ {
+ str.append( format( "%02x", aB ) );
+ }
+ return str.toString() ;
+ }
+
+ public static int compareByKey(Record record1, Record record2)
+ {
+ checkKeyCompatible(record1, record2) ;
+ return Bytes.compare(record1.key, record2.key) ;
+ }
+
+ public static int compareByKeyValue(Record record1, Record record2)
+ {
+ checkCompatible(record1, record2) ;
+ int x = Bytes.compare(record1.key, record2.key) ;
+ if ( x == 0 )
+ {
+ if ( record1.value != null )
+ x = Bytes.compare(record1.value, record2.value) ;
+ }
+ return x ;
+ }
+
+ static void checkCompatible(Record record1, Record record2)
+ {
+ if ( ! compatible(record1, record2, true) )
+ throw new RecordException(format("Incompatible: %s, %s", record1, record2)) ;
+ }
+
+ static void checkKeyCompatible(Record record1, Record record2)
+ {
+ if ( ! compatible(record1, record2, false) )
+ throw new RecordException(format("Incompatible: %s, %s", record1, record2)) ;
+ }
+
+ static boolean compatible(Record record1, Record record2, boolean checkValue)
+ {
+ if ( record1.key.length != record2.key.length )
+ return false ;
+
+ if ( checkValue )
+ {
+ if ( record1.value == null && record2.value == null )
+ return true ;
+ if ( record1.value == null )
+ return false ;
+ if ( record2.value == null )
+ return false ;
+ if ( record1.value.length != record2.value.length )
+ return false;
+ }
+ return true ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordException.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordException.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordException.java
new file mode 100644
index 0000000..4b0301a
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.record;
+
+import org.seaborne.dboe.DBOpEnvException ;
+
+public class RecordException extends DBOpEnvException
+{
+ public RecordException() {}
+ public RecordException(String msg) { super(msg) ; }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordFactory.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordFactory.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordFactory.java
new file mode 100644
index 0000000..a4c9cbb
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordFactory.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.record;
+
+import static java.lang.String.format;
+
+import java.nio.ByteBuffer;
+
+/** Record creator */
+final
+public class RecordFactory
+{
+ private final int keyLength;
+ private final int valueLength;
+ private final int slotLen;
+ private final boolean checking = false;
+
+ public RecordFactory(int keyLength, int valueLength) {
+ if ( keyLength <= 0 )
+ throw new IllegalArgumentException("Bad key length: " + keyLength);
+ if ( valueLength < 0 )
+ throw new IllegalArgumentException("Bad value length: " + valueLength);
+
+ this.keyLength = keyLength;
+ this.valueLength = valueLength;
+ this.slotLen = keyLength + valueLength;
+ }
+
+ /** Return a RecordFactory that makes key-only records of the same key size */
+ public RecordFactory keyFactory() {
+ return new RecordFactory(keyLength, 0);
+ }
+
+ /** Create a key-only record, allocating blank space for the key */
+ public Record createKeyOnly() {
+ return create(new byte[keyLength], null);
+ }
+
+ /** Create a key-only record */
+ public Record createKeyOnly(Record record) {
+ checkKey(record.getKey());
+ if ( record.getValue() == null )
+ return record;
+
+ return create(record.getKey(), null);
+ }
+
+ /** Create a key and value record (value uninitialized) */
+ public Record create(byte[] key) {
+ checkKey(key);
+ byte[] v = null;
+ if ( valueLength > 0 )
+ v = new byte[valueLength];
+ return create(key, v);
+ }
+
+ /** Create a record, allocating space for the key and value (if any) */
+ public Record create() {
+ return create(new byte[keyLength], (valueLength > 0) ? new byte[valueLength] : null);
+ }
+
+ /** Create a key and value record */
+ public Record create(byte[] key, byte[] value) {
+ check(key, value);
+ return new Record(key, value);
+ }
+
+ public void insertInto(Record record, ByteBuffer bb, int idx) {
+ check(record);
+ bb.position(idx * slotLen);
+ bb.put(record.getKey(), 0, keyLength);
+ if ( hasValue() && record.getValue() != null )
+ bb.put(record.getValue(), 0, valueLength);
+ }
+
+ public static final RecordMapper<Record> mapperRecord = new RecordMapper<Record>() {
+ /*
+ * RecordRangeIterator calls getRecordBuffer().get(slotidx);
+ */
+ @Override
+ public Record map(ByteBuffer bb, int idx, byte[] keyBytes, RecordFactory factory) {
+ byte[] key = new byte[factory.keyLength];
+ byte[] value = (factory.hasValue() ? new byte[factory.valueLength] :null );
+ // Is it better to avoid the synchronize needed for bb.position./bb.get
+ // but not use a (relative) bulk bb.get(byte[],,) which may be a native operation?
+
+// int posnKey = idx*slotLen;
+// // Avoid using position() so we can avoid needing synchronized.
+// copyInto(key, bb, posnKey, keyLength);
+// if ( value != null )
+// {
+// int posnValue = idx*slotLen+keyLength;
+// copyInto(value, bb, posnValue, valueLength);
+// }
+
+ // Using bb.get(byte[],,) may be potentially faster but requires the synchronized
+ // There's no absolute version.
+ synchronized(bb)
+ {
+ bb.position(idx*factory.slotLen);
+ bb.get(key, 0, factory.keyLength);
+ if ( value != null )
+ bb.get(value, 0, factory.valueLength);
+ }
+ if ( keyBytes != null )
+ System.arraycopy(key, 0, keyBytes, 0, factory.keyLength);
+ return factory.create(key, value);
+ }
+ };
+
+ public <X> X access(ByteBuffer bb, int idx, byte[] keyBytes, RecordMapper<X> mapper) {
+ return mapper.map(bb, idx, keyBytes, this);
+ }
+
+ public Record buildFrom(ByteBuffer bb, int idx) {
+ return access(bb, idx, null, mapperRecord);
+ }
+
+ private final void copyInto(byte[] dst, ByteBuffer src, int start, int length) {
+ // Thread safe.
+ for ( int i = 0; i < length; i++ )
+ dst[i] = src.get(start+i);
+ // Would otherwise be ...
+// src.position(start);
+// src.get(dst, 0, length);
+ }
+
+ public boolean hasValue() { return valueLength > 0; }
+
+ public int recordLength() { return keyLength + valueLength; }
+
+ public int keyLength() { return keyLength; }
+
+ public int valueLength() { return valueLength; }
+
+ @Override
+ public String toString() {
+ return format("<RecordFactory k=%d v=%d>", keyLength, valueLength);
+ }
+
+ private final void check(Record record) {
+ if ( ! checking ) return;
+ check(record.getKey(), record.getValue());
+ }
+
+ private final void checkKey(byte[] k) {
+ if ( ! checking ) return;
+ if ( k == null )
+ throw new RecordException("Null key byte[]");
+ if ( keyLength != k.length )
+ throw new RecordException(format("Key length error: This RecordFactory manages records of key length %d, not %d", keyLength,
+ k.length));
+ }
+
+ private final void check(byte[] k, byte[] v) {
+ if ( ! checking ) return;
+ checkKey(k);
+ if ( valueLength <= 0 ) {
+ if ( v != null )
+ throw new RecordException("Value array error: This RecordFactory manages records that are all key");
+ } else {
+ // v == null for a key-only record from this factory.
+ if ( v != null && v.length != valueLength )
+ throw new RecordException(format("This RecordFactory manages record of value length %d, not (%d,-)", valueLength,
+ v.length));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordMapper.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordMapper.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordMapper.java
new file mode 100644
index 0000000..faf3037
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordMapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.record;
+
+import java.nio.ByteBuffer ;
+
+/** Allow bytes to pull directly out of storage with no copy to record;
+ * also extract the key bytes into an array.
+ * This operation MUST not retain any references to the ByteBuffer storage space.
+ * All in order to avoid remapping Records to higher level objects,
+ * which churns objects (GC issue) and avoids a copy.
+ */
+public interface RecordMapper<X> {
+ X map(ByteBuffer bb, int entryIdx, byte key[], RecordFactory recFactory) ;
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPage.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPage.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPage.java
new file mode 100644
index 0000000..db93043
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPage.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.recordbuffer;
+
+import org.seaborne.dboe.base.block.Block ;
+import org.seaborne.dboe.base.page.Page ;
+import org.seaborne.dboe.base.record.RecordFactory ;
+import org.seaborne.dboe.sys.Sys ;
+
+/**
+ * B+Tree records nodes and hash buckets.
+ * Add link field to a RecordBufferPageBase
+ */
+
+public final class RecordBufferPage extends RecordBufferPageBase
+{
+ // Why not straight to BPlusTreeRecords?
+ // 1 - this may be useful in its own right as a sequence of records on-disk.
+ // 2 - BPlusTreeRecords inherits from BPlusTreePage
+
+
+ // To Constants
+ // Offsets
+ // final public static int COUNT = 0 ;
+ // Adds this field over RecordBufferPageBase
+ final public static int LINK = 4 ;
+ final private static int FIELD_LENGTH = Sys.SizeOfInt ; // Length of the space needed here (not count)
+
+ private int link = Page.NO_ID ;
+
+ public final int getLink() { return link ; }
+
+ public void setLink(int link)
+ {
+ this.link = link ;
+ getBackingBlock().getByteBuffer().putInt(LINK, link) ;
+ }
+
+ @Override
+ protected void _reset(Block block)
+ {
+ // TODO -- should this be format?
+ // Print this
+ super.rebuild(block, this.getCount()) ;
+ //?? use .format
+// // TODO WRONG : block is overlying.
+// this.link = block.getByteBuffer().getInt(LINK) ;
+ }
+
+ public static int calcRecordSize(RecordFactory factory, int blkSize)
+ { return RecordBufferPageBase.calcRecordSize(factory, blkSize, FIELD_LENGTH) ; }
+
+ public static int calcBlockSize(RecordFactory factory, int maxRec)
+ { return RecordBufferPageBase.calcBlockSize(factory, maxRec, FIELD_LENGTH) ; }
+
+ /** The construction methods */
+ public static RecordBufferPage createBlank(Block block,RecordFactory factory)
+ {
+ int count = 0 ;
+ int linkId = NO_ID ;
+ return new RecordBufferPage(block, factory, count, linkId) ;
+ }
+
+ public static RecordBufferPage format(Block block, RecordFactory factory)
+ {
+ int count = block.getByteBuffer().getInt(COUNT) ;
+ int linkId = block.getByteBuffer().getInt(LINK) ;
+ return new RecordBufferPage(block, factory, count, linkId) ;
+ }
+
+ private RecordBufferPage(Block block, RecordFactory factory, int count, int linkId)
+ {
+ super(block, FIELD_LENGTH, factory, count) ;
+ this.link = linkId ;
+ }
+
+ @Override
+ public String toString()
+ { return String.format("RecordBufferPage[id=%d,link=%d]: %s", getBackingBlock().getId(), getLink(), recBuff) ; }
+
+ @Override
+ public String getRefStr() {
+ return String.format("RecordBufferPage[id=%d]", getBackingBlock().getId()) ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageBase.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageBase.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageBase.java
new file mode 100644
index 0000000..0390e3b
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageBase.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.recordbuffer ;
+
+import java.nio.ByteBuffer ;
+
+import org.apache.jena.atlas.io.IndentedWriter ;
+import org.seaborne.dboe.base.block.Block ;
+import org.seaborne.dboe.base.buffer.RecordBuffer ;
+import org.seaborne.dboe.base.page.PageBase ;
+import org.seaborne.dboe.base.record.RecordFactory ;
+import org.seaborne.dboe.sys.Sys ;
+
+/**
+ * The on-disk form of a block of a single RecordBuffer
+ * (i.e. this is not part of a BTree/BPlusTree branch node).
+ * This must be compatible with B+Tree records nodes and hashbuckets.
+ */
+
+public abstract class RecordBufferPageBase extends PageBase // implements Page
+{
+ // Field offsets
+ final public static int COUNT = 0 ;
+ // Length due to this class - subclasses may use more overhead.
+ final private static int FIELD_LENGTH = Sys.SizeOfInt ;
+
+ protected final int headerLength ;
+
+ // Interface: "Page" - id, byteBuffer, count
+ protected RecordBuffer recBuff ;
+ private final RecordFactory factory ;
+
+ // private int offset ; // Bytes of overhead.
+
+ public static int calcRecordSize(RecordFactory factory, int blkSize, int headerOffset) {
+ // Length = X*recordLength + HEADER
+ int x = blkSize - totalOffset(headerOffset) ;
+ return x / factory.recordLength() ;
+ }
+
+ public static int calcBlockSize(RecordFactory factory, int maxRec, int headerOffset) {
+ return totalOffset(headerOffset) + factory.recordLength() * maxRec ;
+ }
+
+ private static int totalOffset(int headerOffset) {
+ return FIELD_LENGTH + headerOffset ;
+ }
+
+ protected RecordBufferPageBase(Block block, int offset, RecordFactory factory, int count) {
+ // This code knows the alignment of the records in the ByteBuffer.
+ super(block) ;
+ this.headerLength = FIELD_LENGTH + offset ; // NB +4 for the count field
+ this.factory = factory ;
+ rebuild(block, count) ;
+ }
+
+ protected void rebuild(Block block, int count) {
+ ByteBuffer bb = block.getByteBuffer() ;
+ bb.clear() ;
+ bb.position(headerLength) ;
+ bb = bb.slice() ;
+ this.recBuff = new RecordBuffer(bb, factory, count) ;
+ }
+
+ public final RecordBuffer getRecordBuffer() {
+ return recBuff ;
+ }
+
+ public final int getCount() {
+ return recBuff.size() ;
+ }
+
+ public final int getMaxSize() {
+ return recBuff.maxSize() ;
+ }
+
+ public void setCount(int count) {
+ recBuff.setSize(count) ;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("RecordBufferPageBase[id=%d]: %s", getBackingBlock().getId(), recBuff) ;
+ }
+
+ @Override
+ public void output(IndentedWriter out) {
+ out.print(toString()) ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageMgr.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageMgr.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageMgr.java
new file mode 100644
index 0000000..015a5d6
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageMgr.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.recordbuffer;
+
+import static org.seaborne.dboe.base.recordbuffer.RecordBufferPage.* ;
+
+import java.nio.ByteBuffer;
+
+import org.seaborne.dboe.base.block.Block ;
+import org.seaborne.dboe.base.block.BlockMgr ;
+import org.seaborne.dboe.base.block.BlockType ;
+import org.seaborne.dboe.base.page.BlockConverter ;
+import org.seaborne.dboe.base.page.PageBlockMgr ;
+import org.seaborne.dboe.base.record.RecordException ;
+import org.seaborne.dboe.base.record.RecordFactory ;
+
+/** Manager for a block that is all records.
+ * This must be compatible with B+Tree records nodes and with hash buckets.
+ */
+
+final
+public class RecordBufferPageMgr extends PageBlockMgr<RecordBufferPage>
+{
+ private final RecordFactory factory ;
+
+ public RecordBufferPageMgr(RecordFactory factory, BlockMgr blockMgr) {
+ super(new Block2RecordBufferPage(factory), blockMgr) ;
+ this.factory = factory ;
+ }
+
+ public RecordFactory getRecordFactory() { return factory ; }
+
+ public RecordBufferPage create() {
+ return super.create(BlockType.RECORD_BLOCK) ;
+ }
+
+ public static class Block2RecordBufferPage implements BlockConverter<RecordBufferPage> {
+ private RecordFactory factory ;
+
+ public Block2RecordBufferPage(RecordFactory factory) {
+ this.factory = factory ;
+ }
+
+ @Override
+ public RecordBufferPage createFromBlock(Block block, BlockType blkType) {
+ if ( blkType != BlockType.RECORD_BLOCK )
+ throw new RecordException("Not RECORD_BLOCK: " + blkType) ;
+ // Initially empty
+ RecordBufferPage rb = RecordBufferPage.createBlank(block, factory) ;
+ return rb ;
+ }
+
+ @Override
+ public RecordBufferPage fromBlock(Block block) {
+ synchronized (block) // [[TxTDB:TODO] needed? Right place?
+ {
+ RecordBufferPage rb = RecordBufferPage.format(block, factory) ;
+ // int count = block.getByteBuffer().getInt(COUNT) ;
+ // int linkId = block.getByteBuffer().getInt(LINK) ;
+ // RecordBufferPage rb = new RecordBufferPage(block, linkId,
+ // factory, count) ;
+ return rb ;
+ }
+ }
+
+ @Override
+ public Block toBlock(RecordBufferPage rbp) {
+ int count = rbp.getRecordBuffer().size() ;
+ ByteBuffer bb = rbp.getBackingBlock().getByteBuffer() ;
+ bb.putInt(COUNT, rbp.getCount()) ;
+ bb.putInt(LINK, rbp.getLink()) ;
+ return rbp.getBackingBlock() ;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordRangeIterator.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordRangeIterator.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordRangeIterator.java
new file mode 100644
index 0000000..0ec19c3
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordRangeIterator.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.base.recordbuffer;
+
+import static org.apache.jena.atlas.lib.Alg.decodeIndex ;
+
+import java.util.Iterator ;
+import java.util.NoSuchElementException ;
+
+import org.apache.jena.atlas.lib.Bytes ;
+import org.apache.jena.atlas.lib.Closeable ;
+import org.seaborne.dboe.base.StorageException ;
+import org.seaborne.dboe.base.block.BlockException ;
+import org.seaborne.dboe.base.record.Record ;
+import org.seaborne.dboe.base.record.RecordMapper ;
+
+final public
+class RecordRangeIterator<X> implements Iterator<X>, Closeable
+{
+ /** Iterate over a range of fromRec (inclusive) to toRec (exclusive) */
+ public static <X> Iterator<X> iterator(int pageId, Record fromRec, Record toRec, RecordBufferPageMgr pageMgr, RecordMapper<X> mapper)
+ {
+ if ( ! pageMgr.valid(pageId) ) {
+ String msg = "RecordRangeIterator.iterator -- No such block (pageId="+pageId+", fromRec="+fromRec+", toRec="+toRec+ ")" ;
+// System.err.println(msg) ;
+// System.exit(0) ;
+ throw new BlockException(msg) ;
+ }
+ return new RecordRangeIterator<>(pageId, fromRec, toRec, pageMgr, mapper) ;
+ }
+
+ private final RecordMapper<X> mapper ;
+
+ private RecordBufferPage currentPage ; // Set null when finished.
+
+ private int currentIdx ;
+ private final RecordBufferPageMgr pageMgr ;
+ private final Record maxRec ;
+ private final Record minRec ;
+
+ private X slot = null ;
+ private final byte[] keySlot ;
+
+ private long countRecords = 0 ;
+ private long countBlocks = 0 ;
+
+ private RecordRangeIterator(int id, Record fromRec, Record toRec, RecordBufferPageMgr pageMgr, RecordMapper<X> mapper) {
+ currentIdx = 0 ;
+ this.pageMgr = pageMgr;
+ this.minRec = fromRec ;
+ this.maxRec = toRec ;
+ this.mapper = mapper ;
+ this.keySlot = new byte[pageMgr.getRecordFactory().keyLength()] ;
+
+ if ( toRec != null && fromRec != null && Record.keyLE(toRec, fromRec) ) {
+ currentPage = null ;
+ return ;
+ }
+
+ pageMgr.startRead();
+ currentPage = pageMgr.getRead(id) ;
+
+ if ( currentPage.getCount() == 0 ) {
+ // Empty page.
+ close() ;
+ return ;
+ }
+
+ if ( fromRec != null ) {
+ currentIdx = currentPage.getRecordBuffer().find(fromRec) ;
+ if ( currentIdx < 0 )
+ currentIdx = decodeIndex(currentIdx) ;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if ( slot != null )
+ return true ;
+ if ( currentPage == null )
+ return false ;
+ // Set slot.
+ while (currentIdx >= currentPage.getCount()) {
+ // Move to next.
+ int link = currentPage.getLink() ;
+ if ( link < 0 ) {
+ close() ;
+ return false ;
+ }
+
+ if ( currentPage != null )
+ pageMgr.release(currentPage) ;
+
+ RecordBufferPage nextPage = pageMgr.getRead(link) ;
+ // Check currentPage -> nextPage is strictly increasing keys.
+ if ( false ) {
+ Record r1 = currentPage.getRecordBuffer().getHigh() ;
+ Record r2 = nextPage.getRecordBuffer().getLow() ;
+ if ( Record.keyGE(r1, r2) )
+ throw new StorageException("RecordRangeIterator: records not strictly increasing: " + r1 + " // " + r2) ;
+ }
+ currentPage = nextPage ;
+ countBlocks++ ;
+ currentIdx = 0 ;
+ }
+
+ slot = currentPage.getRecordBuffer().access(currentIdx, keySlot, mapper) ;
+ currentIdx++ ;
+
+ if ( maxRec != null && Bytes.compare(keySlot, maxRec.getKey()) >= 0 ) {
+ close() ;
+ return false ;
+ }
+
+ if ( slot == null ) {
+ close() ;
+ return false ;
+ }
+ countRecords++ ;
+ return true ;
+ }
+
+ @Override
+ public void close() {
+ if ( currentPage == null )
+ return ;
+ pageMgr.release(currentPage) ;
+ pageMgr.finishRead();
+ currentPage = null ;
+ currentIdx = -99 ;
+ slot = null ;
+ }
+
+ @Override
+ public X next() {
+ if ( !hasNext() )
+ throw new NoSuchElementException() ;
+ X x = slot ;
+ slot = null ;
+ return x ;
+ }
+
+ final public long getCountRecords() { return countRecords ; }
+
+ final public long getCountBlocks() { return countBlocks ; }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/jenax/Txn.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/jenax/Txn.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/jenax/Txn.java
new file mode 100644
index 0000000..c91438b
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/jenax/Txn.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.jenax;
+
+import java.util.function.Supplier ;
+
+import org.apache.jena.sparql.core.Transactional ;
+
+/** Indirection to allow for modification for TDB2 capabilities */
+public class Txn {
+ public static <T extends Transactional> void executeRead(T txn, Runnable r) {
+ org.apache.jena.system.Txn.executeRead(txn, r);
+ }
+
+ public static <T extends Transactional, X> X calculateRead(T txn, Supplier<X> r) {
+ return org.apache.jena.system.Txn.calculateRead(txn, r);
+ }
+
+ public static <T extends Transactional> void executeWrite(T txn, Runnable r) {
+ org.apache.jena.system.Txn.executeWrite(txn, r);
+ }
+
+ public static <T extends Transactional, X> X calculateWrite(T txn, Supplier<X> r) {
+ return org.apache.jena.system.Txn.calculateWrite(txn, r);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/migrate/L.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/migrate/L.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/migrate/L.java
new file mode 100644
index 0000000..c3c6581
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/migrate/L.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.migrate;
+
+import java.io.IOException ;
+import java.io.OutputStream ;
+import java.io.OutputStreamWriter ;
+import java.io.Writer ;
+import java.nio.ByteBuffer ;
+import java.nio.charset.StandardCharsets ;
+import java.util.UUID ;
+import java.util.concurrent.* ;
+import java.util.concurrent.locks.Lock ;
+import java.util.function.Supplier ;
+
+import org.apache.jena.atlas.io.IO ;
+import org.apache.jena.atlas.lib.Bytes ;
+import org.apache.jena.atlas.lib.StrUtils ;
+import org.apache.jena.shared.uuid.JenaUUID ;
+
+/** Misc class */
+public class L {
+
+ // Not to be confused with UUID.nameUUIDFromBytes (a helper for version 3 UUIDs)
+ /**
+ * Java UUID to bytes (most significant first)
+ */
+ public static byte[] uuidAsBytes(UUID uuid) {
+ return uuidAsBytes(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()) ;
+ }
+
+ /**
+ * Jena UUID to bytes (most significant first)
+ */
+ public static byte[] uuidAsBytes(JenaUUID uuid) {
+ return uuidAsBytes(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()) ;
+ }
+
+ /** UUID, as two longs, as bytes */
+ public static byte[] uuidAsBytes(long mostSignificantBits, long leastSignificantBits) {
+ byte[] bytes = new byte[16] ;
+ Bytes.setLong(mostSignificantBits, bytes, 0);
+ Bytes.setLong(leastSignificantBits, bytes, 8);
+ return bytes ;
+ }
+
+ /** A UUID string to bytes */
+ public static byte[] uuidAsBytes(String str) {
+ return uuidAsBytes(UUID.fromString(str)) ;
+ }
+
+ public static String uuidToString(long mostSignificantBits, long leastSignificantBits) {
+ return new UUID(mostSignificantBits, leastSignificantBits).toString() ;
+ //JenaUUID.toString(mostSignificantBits, leastSignificantBits)
+ }
+
+ private static ExecutorService executor = Executors.newCachedThreadPool() ;
+
+ /**
+ * Run asynchronously on another thread ; the thread has started
+ * when this function returns.
+ */
+ public static void async(Runnable r) {
+ Semaphore semaStart = new Semaphore(0, true) ;
+ Runnable r2 = () -> {
+ semaStart.release(1) ;
+ r.run();
+ } ;
+ executor.execute(r2);
+ semaStart.acquireUninterruptibly();
+ }
+
+ /** Run synchronously but on another thread. */
+ public static void syncOtherThread(Runnable r) {
+ runCallable(()->{
+ r.run();
+ return null ;
+ }) ;
+ }
+
+ /** Run synchronously but on another thread. */
+ public static <T> T syncCallThread(Supplier<T> r) {
+ return runCallable(() -> {
+ T t = r.get() ;
+ return t ;
+ }) ;
+ }
+
+ private static <T> T runCallable(Callable<T> action) {
+ try { return executor.submit(action).get() ; }
+ catch (Exception e) {
+ e.printStackTrace();
+ return null ;
+ }
+ }
+
+
+ /** Execute. Preform the "before" action, then main action.
+ * Always call the "after" runnable if the "bafore" succeeded.
+ * Becareful about argument order.
+ * @param action
+ * @param before
+ * @param after
+ */
+ public static void withBeforeAfter(Runnable action, Runnable before, Runnable after) {
+ before.run();
+ try { action.run(); }
+ finally { after.run(); }
+ }
+
+ /** Execute. Preform the "before" action, then main action.
+ * Always call the "after" runnable if the "bafore" succeeded.
+ * Becareful about argument order.
+ * @param action
+ * @param before
+ * @param after
+ */
+ public static <V> V callWithBeforeAfter(Supplier<V> action, Runnable before, Runnable after) {
+ before.run();
+ try { return action.get() ; }
+ finally { after.run(); }
+ }
+
+ /** Execute; always call the "after" runnable */
+ public static void withAfter(Runnable action, Runnable after) {
+ try { action.run(); }
+ finally { after.run(); }
+ }
+
+ /** Execute and return a value; always call the "after" runnable */
+ public static <V> V callWithAfter(Supplier<V> action, Runnable after) {
+ try { return action.get() ; }
+ finally { after.run(); }
+ }
+
+ /** Run inside a Lock */
+ public static <V> V callWithLock(Lock lock, Supplier<V> r) {
+ return callWithBeforeAfter(r, ()->lock.lock(), ()->lock.unlock()) ;
+ }
+
+ /** Run inside a Lock */
+ public static void withLock(Lock lock, Runnable r) {
+ withBeforeAfter(r, ()->lock.lock(), ()->lock.unlock()) ;
+ }
+
+ // ==> IO.writeWholeFileAsUTF8
+
+ /** Write a string to a file as UTF-8. The file is closed after the operation.
+ * @param filename
+ * @param content String to be writtem
+ * @throws IOException
+ */
+
+ public static void writeStringAsUTF8(String filename, String content) throws IOException {
+ try ( OutputStream out = IO.openOutputFileEx(filename) ) {
+ writeStringAsUTF8(out, content) ;
+ out.flush() ;
+ }
+ }
+
+ /** Read a whole stream as UTF-8
+ *
+ * @param out OutputStream to be read
+ * @param content String to be written
+ * @throws IOException
+ */
+ public static void writeStringAsUTF8(OutputStream out, String content) throws IOException {
+ Writer w = new OutputStreamWriter(out, StandardCharsets.UTF_8) ;
+ w.write(content);
+ w.flush();
+ // Not close.
+ }
+
+ // ==> IO.writeWholeFileAsUTF8
+
+ /** String to ByteBuffer */
+ public static ByteBuffer stringToByteBuffer(String str) {
+ byte[] b = StrUtils.asUTF8bytes(str) ;
+ return ByteBuffer.wrap(b) ;
+ }
+
+ /** ByteBuffer to String */
+ public static String byteBufferToString(ByteBuffer bb) {
+ byte[] b = new byte[bb.remaining()] ;
+ bb.get(b) ;
+ return StrUtils.fromUTF8bytes(b) ;
+ }
+
+
+}
+
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/FileLib.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/FileLib.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/FileLib.java
new file mode 100644
index 0000000..8293d94
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/FileLib.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.sys;
+
+import java.io.FileNotFoundException ;
+import java.io.IOException ;
+import java.io.RandomAccessFile ;
+import java.nio.channels.FileChannel ;
+
+import org.apache.jena.atlas.io.IO ;
+import org.seaborne.dboe.base.file.ChannelManager ;
+import org.seaborne.dboe.base.file.FileException ;
+
+// => IO
+public class FileLib
+{
+ // Check whether still used.
+
+ public static FileChannel openUnmanaged(String filename) {
+ return openUnmanaged(filename, "rw") ;
+ }
+
+ // And operation from BufferChannelFile
+
+ public static FileChannel openUnmanaged(String filename, String mode) {
+ try {
+ @SuppressWarnings("resource")
+ RandomAccessFile out = new RandomAccessFile(filename, mode) ;
+ FileChannel channel = out.getChannel() ;
+ return channel ;
+ }
+ catch (FileNotFoundException e) {
+ IO.exception(e) ;
+ return null ;
+ }
+ }
+
+ // TODO remove and call ChannelManager directly
+ public static FileChannel openManaged(String filename) {
+ return openManaged(filename, "rw") ;
+ }
+
+ // TODO remove and call ChannelManager directly
+ public static FileChannel openManaged(String filename, String mode) {
+ return ChannelManager.acquire(filename, mode) ;
+ }
+
+ public static long size(FileChannel channel) {
+ try {
+ return channel.size() ;
+ } catch (IOException ex)
+ { IO.exception(ex) ; return -1L ; }
+ }
+
+ public static void truncate(FileChannel channel, long posn) {
+ try { channel.truncate(posn) ; }
+ catch (IOException ex) { IO.exception(ex) ; }
+ }
+
+ public static void close(FileChannel channel) {
+ ChannelManager.release(channel) ;
+ }
+
+ public static void sync(FileChannel channel) {
+ try {
+ channel.force(true) ;
+ } catch (IOException ex)
+ { throw new FileException("FileBase.sync", ex) ; }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Names.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Names.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Names.java
new file mode 100644
index 0000000..14b9fd6
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Names.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.sys;
+
+/** Names of things in TDB */
+public class Names
+{
+ /** Filename of the TDB configuration file */
+ public static final String TDB_CONFIG_FILE = "tdb.cfg" ;
+
+ /** Filename of the TDB lock file */
+ public static final String TDB_LOCK_FILE = "tdb.lock" ;
+
+ /* System files */
+ public static final String extJournal = "jrnl" ;
+ /* Component Transactional State */
+ public static final String extState = "cts" ;
+
+ public static final String journalFileBase = "journal" ;
+ public static final String journalFile = journalFileBase+"."+extJournal ;
+
+ // ++ Elsewhere
+ /* RDF indexes */
+ public static final String primaryIndexTriples = "SPO" ;
+ public static final String[] tripleIndexes = { primaryIndexTriples, "POS", "OSP" } ;
+
+ public static final String primaryIndexQuads = "GSPO" ;
+ public static final String[] quadIndexes = { primaryIndexQuads, "GPOS", "GOSP", "POSG", "OSPG", "SPOG"} ;
+
+ public static final String primaryIndexPrefix = "GPU" ;
+ public static final String[] prefixIndexes = { primaryIndexPrefix } ;
+ // ++ Elsewhere
+
+ /* B+Trees - nodes file and records file */
+ public static final String extBptState = "bpt" ;
+ public static final String extBptTree = "idn" ;
+ public static final String extBptRecords = "dat" ;
+
+// /* B+Trees - single file per tree */
+// public static final String btExt = "idx" ;
+
+ /* ExtHash - dictionary file*/
+ public static final String extHashExt = "exh" ;
+ public static final String extHashBucketExt = "dat" ;
+
+ public static final String datasetConfig = "config-tdb" ; // name of the TDB configuration file.
+
+ /* Node files */
+ public static final String extBdfState = "bdf" ; // Node state
+ public static final String extObjNodeData = "obj" ;
+ public static final String nodeTableBaseName = "nodes" ; // Node table
+
+ /** Prefixes file */
+ public static final String prefixTableBaseName = "prefixes" ; // Prefix node table
+
+ /** Optimizer / stats */
+ public static final String optStats = "stats.opt" ;
+ public static final String optFixed = "fixed.opt" ;
+ public static final String optNone = "none.opt" ;
+ public static final String optDefault = optFixed ;
+
+ public static final String extMeta = "info" ;
+ public static final String directoryMetafile = "this" ; // Root name of the directory for a metafile.
+
+ /** Name to indicate in-memory */
+ public static final String memName = "--mem--" ;
+
+ public static boolean isMem(String name) { return memName.equals(name) ; }
+
+ // ---- Names for Java properties in metadata files
+
+ public static String makeKey(String root, String...elements)
+ { return makeName(root, elements) ; }
+
+ public static String makeName(String root, String...elements)
+ {
+ StringBuilder sb = new StringBuilder() ;
+ sb.append(root) ;
+ for ( String s : elements )
+ {
+ if ( ! s.startsWith(".") )
+ sb.append(".") ;
+ sb.append(s) ;
+ }
+ return sb.toString() ;
+ }
+
+ // Component elements.
+ public static final String elNode = "node" ;
+ public static final String elBPlusTree = "bptree" ;
+ public static final String elIndex = "index" ;
+ public static final String elObject = "object" ;
+
+ public static final String elType = "type" ;
+ public static final String elLayout = "layout" ;
+ public static final String elVersion = "version" ;
+ public static final String elTimestamp = "timestamp" ;
+ public static final String elCreated = "created" ;
+
+ // Root names.
+ public static final String keyNS = "tdb" ;
+ public static final String keyNSNode = makeName(keyNS, elNode) ;
+ public static final String keyNSBPlusTree = makeName(keyNS, elBPlusTree) ;
+
+ // Location metadata - in the directory wide metadata file.
+ public static final String kVersion = makeName(keyNS, elVersion) ;
+ public static final String kCreatedDate = makeName(keyNS, elTimestamp) ;
+
+ // Node table metadata
+ public static final String kNodeTableType = makeName(keyNS, elNode, elType) ;
+ public static final String kNodeTableLayout = makeName(keyNS, elNode, elLayout) ;
+
+// // Object file
+// public static final String kObjectTableType = makeName(keyNS, elObject, elType) ;
+//
+// // Default index metadata - in the index metadata file.
+//
+// public static final String kIndexType = makeName(keyNS, elIndex, elType) ;
+// public static final String kIndexFileLayout = makeName(keyNS, elIndex, elLayout) ;
+
+ // See also BPlusTreeParams for keyNSBPlusTree derived names.
+
+// // Current values
+// public static final String currentIndexType = IndexType.BPlusTree.getName() ;
+// public static final String currentIndexFileVersion = "v1" ;
+
+ // Property strings - Setup
+
+ public static final String pNode2NodeIdCacheSize = "tdb.cache.node2nodeid.size" ;
+ public static final String pNodeId2NodeCacheSize = "tdb.cache.nodeid2node.size" ;
+ public static final String pNodeMissesCacheSize = "tdb.cache.nodeMiss.size" ;
+ public static final String pBlockWriteCacheSize = "tdb.cache.blockwrite.size" ;
+ public static final String pBlockReadCacheSize = "tdb.cache.blockread.size" ;
+ public static final String pSyncTick = "tdb.synctick" ;
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/ProcessUtils.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/ProcessUtils.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/ProcessUtils.java
new file mode 100644
index 0000000..2741fad
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/ProcessUtils.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.sys;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ProcessUtils {
+
+ private static int myPid = -1;
+
+ /**
+ * Tries to get the PID of the current process caching it for future calls
+ * since it won't change throughout the life of the process
+ *
+ * @param fallback
+ * Fallback PID to return if unable to determine the PID i.e. an
+ * error code to return
+ *
+ * @return PID of current process or provided {@code fallback} if unable to
+ * determine PID
+ */
+ public static int getPid(int fallback) {
+ if (myPid != -1)
+ return myPid;
+
+ String runtimeBeanName = ManagementFactory.getRuntimeMXBean().getName();
+ if (runtimeBeanName == null) {
+ return useFallbackPid(fallback);
+ }
+
+ // Bean name will have format PID@hostname so we try to parse the PID
+ // portion
+ int index = runtimeBeanName.indexOf("@");
+ if (index < 0)
+ return useFallbackPid(fallback);
+ try {
+ // Parse and cache for future reuse
+ String pidData = runtimeBeanName.substring(0, index);
+ myPid = Integer.parseInt(pidData);
+ return myPid;
+ } catch (NumberFormatException e) {
+ // Invalid PID
+ return useFallbackPid(fallback);
+ }
+ }
+
+ private static int useFallbackPid(int fallback) {
+ // In the case where we can't determine our PID then treat ourselves as
+ // no owner and cache for future use
+ myPid = fallback;
+ return myPid;
+ }
+
+ /**
+ * Determines whether a given PID is alive
+ *
+ * @param pid
+ * PID
+ * @return True if the given PID is alive or unknown, false if it is dead
+ */
+ public static boolean isAlive(int pid) {
+ String pidStr = Integer.toString(pid);
+ try {
+ List<String> data = getProcessInfo(pidStr);
+
+ // Expect a line to contain the PID to indicate the process is
+ // alive
+ for (String lineData : data) {
+ if (lineData.contains(pidStr))
+ return true;
+ }
+
+ // Did not find any lines mentioning the PID so we can safely
+ // assume that process is dead
+ return false;
+ } catch (IOException e) {
+ // If any error running the process to check for the live process
+ // then our check failed and for safety we assume the process is
+ // alive
+
+ Sys.errlog
+ .warn("Your platform does not support checking process liveness so TDB disk locations cannot be reliably locked to prevent possible corruption due to unsafe multi-JVM usage");
+ return true;
+ }
+ }
+
+ /**
+ * Gets whether the platform we are running on will cause us to treat
+ * negative (i.e. invalid) PIDs as alive because of the format in which the
+ * command line process monitor application for the system returns errors
+ * for invalid PIDs
+ *
+ * @return True if a negative PID is treated as alive on this platform or we
+ * cannot determine liveness for PIDs on this platform, false
+ * otherwise
+ */
+ public static boolean negativePidsTreatedAsAlive() {
+ return isAlive(-1);
+ }
+
+ /**
+ * Gets process information based on the given PID string
+ *
+ * @param pidStr
+ * PID string
+ * @return Output of running the OSes appropriate command line task info
+ * application
+ * @throws IOException
+ * Thrown if there is a problem running the command line
+ * application or reading the data returned
+ */
+ private static List<String> getProcessInfo(String pidStr) throws IOException {
+ Process p;
+ if (Sys.isWindows) {
+ // Use the Windows tasklist utility
+ ProcessBuilder builder = new ProcessBuilder("tasklist", "/FI", "PID eq " + pidStr);
+ builder.redirectErrorStream(true);
+ p = builder.start();
+ } else {
+ // Use the ps utility
+ ProcessBuilder builder = new ProcessBuilder("ps", "-p", pidStr);
+ builder.redirectErrorStream(true);
+ p = builder.start();
+ }
+
+ // Run and read data from the process
+ try(BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
+ List<String> data = new ArrayList<>();
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ data.add(line);
+ }
+ return data;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Sys.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Sys.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Sys.java
new file mode 100644
index 0000000..d0264d5
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/Sys.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.sys;
+
+import java.nio.ByteOrder;
+import java.util.concurrent.Executor ;
+import java.util.concurrent.Executors ;
+
+import org.apache.jena.atlas.logging.Log ;
+import org.seaborne.dboe.DBOpEnvException ;
+import org.slf4j.Logger ;
+import org.slf4j.LoggerFactory ;
+
+/** Low level environment */
+public class Sys
+{
+ static final Logger log = LoggerFactory.getLogger("Sys") ;
+
+ /** System log - use for general messages (a few) and warnings.
+ * Generally, do not log events unless you want every user to see them every time.
+ * Libraries and embedded systems should be seen and not heard.
+ * @see #errlog
+ */
+
+ /** General system log */
+ public static final Logger syslog = LoggerFactory.getLogger("System") ;
+ /** Send warnings and error */
+ public static final Logger errlog = LoggerFactory.getLogger("System") ;
+
+ /** Size, in bytes, of a Java long */
+ public static final int SizeOfLong = Long.BYTES ; // Long.SIZE/Byte.SIZE ;
+
+ /** Size, in bytes, of a Java int */
+ public static final int SizeOfInt = Integer.BYTES ; //Integer.SIZE/Byte.SIZE ;
+
+ public static final boolean is64bitSystem = determineIf64Bit() ;
+
+ public static final ByteOrder NetworkOrder = ByteOrder.BIG_ENDIAN ;
+
+ // To make the class initialize
+ static public void init() {}
+
+ public static void panic(Class<? > clazz, String string) {
+ Log.error(clazz, string) ;
+ throw new DBOpEnvException(string) ;
+ }
+ public static final boolean isWindows = determineIfWindows() ; // Memory mapped files behave differently.
+
+ //Or look in File.listRoots.
+ //Alternative method:
+ // http://stackoverflow.com/questions/1293533/name-of-the-operating-system-in-java-not-os-name
+
+ private static boolean determineIfWindows() {
+ String s = System.getProperty("os.name") ;
+ if ( s == null )
+ return false ;
+ return s.startsWith("Windows ") ;
+ }
+
+ /** A general thread pool */
+ public static Executor executor = Executors.newCachedThreadPool() ;
+
+ private static boolean determineIf64Bit() {
+ String s = System.getProperty("sun.arch.data.model") ;
+ if ( s != null ) {
+ boolean b = s.equals("64") ;
+ syslog.debug("System architecture: " + (b ? "64 bit" : "32 bit")) ;
+ return b ;
+ }
+ // Not a SUN VM
+ s = System.getProperty("java.vm.info") ;
+ if ( s == null ) {
+ log.warn("Can't determine the data model") ;
+ return false ;
+ }
+ log.debug("Can't determine the data model from 'sun.arch.data.model' - using java.vm.info") ;
+ boolean b = s.contains("64") ;
+ syslog.debug("System architecture: (from java.vm.info) " + (b ? "64 bit" : "32 bit")) ;
+ return b ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemFile.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemFile.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemFile.java
new file mode 100644
index 0000000..d95ceb8
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemFile.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.sys;
+
+public class SystemFile {
+ public static final int ObjectFileWriteBufferSize = 8*1024 ;
+}
+
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemIndex.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemIndex.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemIndex.java
new file mode 100644
index 0000000..08cfefa
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/sys/SystemIndex.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.sys;
+
+import org.seaborne.dboe.DBOpEnvException ;
+import org.seaborne.dboe.base.block.FileMode ;
+
+public class SystemIndex
+{
+ /** Size, in bytes, of a pointer between blocks */
+ public static final int SizeOfPointer = Sys.SizeOfInt ;
+
+ public static final boolean is64bitSystem = Sys.is64bitSystem ;
+
+ // To make the class initialize
+ static public void init() {}
+
+ /** Size, in bytes, of a block */
+ public static final int BlockSize = 8*1024 ; // intValue("BlockSize", 8*1024) ;
+
+ /** Size, in bytes, of a block for testing */
+ public static final int BlockSizeTest = 1024 ; // intValue("BlockSizeTest", 1024) ;
+
+ /** Size, in bytes, of a block for testing */
+ public static final int BlockSizeTestMem = 500 ;
+
+// /** Size, in bytes, of a memory block */
+// public static final int BlockSizeMem = 32*8 ; //intValue("BlockSizeMem", 32*8 ) ;
+
+ /** Order of an in-memory BTree or B+Tree */
+ public static final int OrderMem = 5 ; // intValue("OrderMem", 5) ;
+
+ /** Size, in bytes, of a segment (used for memory mapped files) */
+ public static final int SegmentSize = 8*1024*1024 ; // intValue("SegmentSize", 8*1024*1024) ;
+
+ // ---- Cache sizes (within the JVM)
+
+ /** Size of Node to NodeId cache.
+ * Used to map from Node to NodeId spaces.
+ * Used for loading and for query preparation.
+ */
+ public static final int Node2NodeIdCacheSize = intValue("Node2NodeIdCacheSize", ( is64bitSystem ? 100*1000 : 50*1000 )) ;
+
+ /** Size of NodeId to Node cache.
+ * Used to map from NodeId to Node spaces.
+ * Used for retriveing results.
+ */
+ public static final int NodeId2NodeCacheSize = intValue("NodeId2NodeCacheSize", ( is64bitSystem ? 500*1000 : 50*1000 ) ) ;
+
+ /** Size of Node lookup miss cache. */
+ public static final int NodeMissCacheSize = 100 ;
+
+ /** Size of the delayed-write block cache (32 bit systems only) (per file) */
+ public static final int BlockWriteCacheSize = intValue("BlockWriteCacheSize", 2*1000) ;
+
+ /** Size of read block cache (32 bit systems only). Increase JVM size as necessary. Per file. */
+ public static final int BlockReadCacheSize = intValue("BlockReadCacheSize", 10*1000) ;
+
+ private static int intValue(String name, int dft) { return dft ; }
+
+
+ public static void setNullOut(boolean nullOut)
+ { NullOut = nullOut ; }
+
+ /** Are we nulling out unused space in bytebuffers (records, points etc) */
+ public static boolean getNullOut()
+ { return NullOut ; }
+
+ /** null out (with the FillByte) freed up space in buffers */
+ public static boolean NullOut = false ;
+
+ /** FillByte value for NullOut */
+ public static final byte FillByte = (byte)0xFF ;
+
+ public static boolean Checking = false ; // This isn't used enough!
+
+ // ---- File mode
+
+ private static FileMode fileMode = null ;
+ public static FileMode fileMode()
+ {
+ if ( fileMode == null )
+ fileMode = determineFileMode() ;
+ return fileMode ;
+ }
+
+ public static void setFileMode(FileMode newFileMode)
+ {
+ if ( fileMode != null )
+ {
+ Sys.log.warn("System file mode already determined - setting it has no effect") ;
+ return ;
+ }
+ fileMode = newFileMode ;
+ }
+
+ // So the test suite can setup thing up ... very carefully.
+ /*package*/ static void internalSetFileMode(FileMode newFileMode)
+ {
+ fileMode = newFileMode ;
+ }
+
+ private static FileMode determineFileMode()
+ {
+ // Be careful that this is not called very, very early, before --set might be seen.
+ // Hence delayed access above in fileMode().
+
+ //String x = ARQ.getContext().getAsString(SystemTDB.symFileMode, "default") ;
+ String x = "default" ;
+
+ if ( x.equalsIgnoreCase("direct") )
+ {
+ Sys.syslog.info("File mode: direct (forced)") ;
+ return FileMode.direct ;
+ }
+ if ( x.equalsIgnoreCase("mapped") )
+ {
+ Sys.syslog.info("File mode: mapped (forced)") ;
+ return FileMode.mapped ;
+ }
+
+ if ( x.equalsIgnoreCase("default") )
+ {
+ if ( is64bitSystem )
+ {
+ Sys.syslog.debug("File mode: Mapped") ;
+ return FileMode.mapped ;
+ }
+ Sys.syslog.debug("File mode: Direct") ;
+ return FileMode.direct ;
+ }
+ throw new DBOpEnvException("Unrecognized file mode (not one of 'default', 'direct' or 'mapped': "+x) ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/BufferTestLib.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/BufferTestLib.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/BufferTestLib.java
new file mode 100644
index 0000000..3e371b3
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/BufferTestLib.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.test;
+
+import java.nio.ByteBuffer ;
+
+import org.seaborne.dboe.base.block.Block ;
+
+public class BufferTestLib
+{
+ public static boolean sameValue(Block block1, Block block2)
+ {
+ if ( block1.getId() != block2.getId()) return false ;
+ ByteBuffer bb1 = block1.getByteBuffer() ;
+ ByteBuffer bb2 = block2.getByteBuffer() ;
+
+ if ( bb1.capacity() != bb2.capacity() ) return false ;
+
+ for ( int i = 0 ; i < bb1.capacity() ; i++ )
+ if ( bb1.get(i) != bb2.get(i) ) return false ;
+ return true ;
+ }
+
+ public static boolean sameValue(ByteBuffer bb1, ByteBuffer bb2)
+ {
+ if ( bb1.capacity() != bb2.capacity() ) return false ;
+
+ int posn1 = bb1.position();
+ int limit1 = bb1.limit();
+ int posn2 = bb2.position();
+ int limit2 = bb2.limit();
+
+ bb1.clear() ;
+ bb2.clear() ;
+
+ try {
+ for ( int i = 0 ; i < bb1.capacity() ; i++ )
+ if ( bb1.get(i) != bb2.get(i) ) return false ;
+ return true ;
+ } finally {
+ bb1.position(posn1) ;
+ bb1.limit(limit1) ;
+ bb2.position(posn2) ;
+ bb2.limit(limit2) ;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/RecordLib.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/RecordLib.java b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/RecordLib.java
new file mode 100644
index 0000000..0c411d9
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/main/java/org/seaborne/dboe/test/RecordLib.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe.test;
+
+
+import java.util.ArrayList ;
+import java.util.Iterator ;
+import java.util.List ;
+
+import org.apache.jena.atlas.iterator.Iter ;
+import org.apache.jena.atlas.lib.Bytes ;
+import org.seaborne.dboe.base.record.Record ;
+import org.seaborne.dboe.base.record.RecordFactory ;
+
+/** Record support operations (mainly for testing using ints) */
+
+public class RecordLib
+{
+ // Size of a record when testing (one integer)
+ public final static int TestRecordLength = 4 ;
+
+ public final static RecordFactory recordFactory = new RecordFactory(TestRecordLength, 0) ;
+
+ public static Record intToRecord(int v) { return intToRecord(v, recordFactory) ; }
+ public static Record intToRecord(int v, int recLen) { return intToRecord(v, new RecordFactory(recLen, 0)) ; }
+
+ public static Record intToRecord(int v, RecordFactory factory)
+ {
+ byte[] vb = Bytes.packInt(v) ;
+
+ int recLen = factory.recordLength() ;
+ byte[] bb = new byte[recLen] ;
+ int x = 0 ; // Start point in bb.
+ if ( recLen > 4 )
+ x = recLen-4 ;
+
+ int len = Math.min(4, recLen) ;
+ int z = 4-len ; // Start point in vb
+
+ // Furthest right bytes.
+ for ( int i = len-1 ; i >= 0 ; i-- )
+ bb[x+i] = vb[z+i] ;
+
+ return factory.create(bb) ;
+ }
+
+ public static List<Record> intToRecord(int[] v) { return intToRecord(v, recordFactory) ; }
+
+ public static List<Record> intToRecord(int[] v, int recLen)
+ { return intToRecord(v, new RecordFactory(recLen, 0)) ; }
+
+ static List<Record> intToRecord(int[] v, RecordFactory factory)
+ {
+ List<Record> x = new ArrayList<>() ;
+ for ( int i : v )
+ x.add(intToRecord(i, factory)) ;
+ return x ;
+ }
+
+ public static int recordToInt(Record key)
+ {
+ return Bytes.getInt(key.getKey()) ;
+ }
+
+ public static List<Integer> toIntList(Iterator<Record> iter)
+ {
+ return Iter.toList(Iter.map(iter, item->recordToInt(item))) ;
+ }
+
+ public static Record r(int v)
+ {
+ return RecordLib.intToRecord(v, RecordLib.TestRecordLength) ;
+ }
+
+ public static int r(Record rec)
+ {
+ return RecordLib.recordToInt(rec) ;
+ }
+
+ public static List<Integer> toIntList(int... vals)
+ {
+ List<Integer> x = new ArrayList<>() ;
+ for ( int i : vals )
+ x.add(i) ;
+ return x ;
+ }
+
+ public static List<Integer> r(Iterator<Record> iter)
+ {
+ return RecordLib.toIntList(iter) ;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/ConfigTestDBOE.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/ConfigTestDBOE.java b/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/ConfigTestDBOE.java
new file mode 100644
index 0000000..2e7fb81
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/ConfigTestDBOE.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe;
+
+import org.apache.jena.atlas.lib.FileOps ;
+import org.seaborne.dboe.sys.Sys ;
+
+public class ConfigTestDBOE
+{
+ private static String testingDataRoot = "testing" ;
+ // Place under target
+ private static final String testingDir = "target/dboe-testing" ;
+ static boolean nonDeleteableMMapFiles = Sys.isWindows ;
+
+ static boolean initialized = false ;
+
+ private static void init()
+ {
+ FileOps.ensureDir("target") ;
+ FileOps.ensureDir(testingDir) ;
+ initialized = true ;
+ }
+
+ private static int count = 0 ;
+
+ public static void setTestingDataRoot(String dir) { testingDataRoot = dir ; }
+ public static String getTestingDataRoot() { return testingDataRoot ; }
+
+ /** return a directory */
+ public static final String getCleanDir() {
+ init() ;
+ String dir = nonDeleteableMMapFiles ? getTestingDirUnique() : getTestingDir() ;
+ FileOps.ensureDir(dir);
+ FileOps.clearDirectory(dir) ;
+ return dir ;
+ }
+ /** Get a empty directory name that has not been used before in this JVM */
+
+ private static final String getTestingDirUnique()
+ {
+ init() ;
+ String dn = testingDir+"/D-"+(++count) ;
+ FileOps.ensureDir(dn) ;
+ FileOps.clearDirectory(dn) ;
+ return dn ;
+ }
+
+ public static final String getTestingDir()
+ {
+ init() ;
+ return testingDir ;
+ }
+
+ public static final void deleteTestingDir()
+ {
+ if ( ! FileOps.exists(testingDir) )
+ return ;
+ FileOps.clearDirectory(testingDir) ;
+ FileOps.deleteSilent(testingDir) ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/1dabea3a/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/TC_DBOE_IO.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/TC_DBOE_IO.java b/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/TC_DBOE_IO.java
new file mode 100644
index 0000000..6a8bd21
--- /dev/null
+++ b/jena-db/jena-dboe-base/src/test/java/org/seaborne/dboe/TC_DBOE_IO.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.seaborne.dboe;
+
+import org.apache.log4j.Level ;
+import org.apache.log4j.Logger ;
+import org.junit.AfterClass ;
+import org.junit.BeforeClass ;
+import org.junit.runner.RunWith ;
+import org.junit.runners.Suite ;
+import org.seaborne.dboe.base.block.FileMode ;
+import org.seaborne.dboe.base.block.TS_Block ;
+import org.seaborne.dboe.base.buffer.TS_Buffer ;
+import org.seaborne.dboe.base.file.TS_File ;
+import org.seaborne.dboe.base.record.TS_Record ;
+import org.seaborne.dboe.base.recordfile.TS_RecordFile ;
+import org.seaborne.dboe.sys.SystemIndex ;
+
+@RunWith(Suite.class)
+@Suite.SuiteClasses( {
+ TS_Block.class
+ , TS_File.class
+ , TS_Buffer.class
+ , TS_Record.class
+ , TS_RecordFile.class
+} )
+
+public class TC_DBOE_IO {
+ static {
+ if ( false )
+ SystemIndex.setFileMode(FileMode.direct) ;
+ }
+
+ @BeforeClass static public void beforeClass()
+ {
+ //org.apache.log4j.LogManager.resetConfiguration() ;
+ //org.apache.log4j.PropertyConfigurator.configure("log4j.properties") ;
+ Logger.getLogger("org.apache.jena.tdb.info").setLevel(Level.WARN) ;
+ //Logger.getLogger("com.hp.hpl.jena.tdb.exec").setLevel(Level.WARN) ;
+ }
+
+ @AfterClass static public void afterClass() {
+ }
+}
+