You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2017/09/28 11:09:05 UTC

[49/50] [abbrv] jena git commit: JENA-1396: Merge //github/afs/mantis as subdirectory jena-db/

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFactory.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFactory.java
index 0000000,0000000..583c490
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFactory.java
@@@ -1,0 -1,0 +1,150 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block ;
++
++import org.apache.jena.atlas.logging.Log ;
++import org.seaborne.dboe.DBOpEnvException ;
++import org.seaborne.dboe.base.file.* ;
++import org.seaborne.dboe.sys.SystemIndex ;
++
++public class BlockMgrFactory {
++    // This isn't always helpful so be careful if setting the default to "true".
++    // Sometimes the tracking is too strict
++    // e.g. transactions keep blocks and not release them down the layers.
++    public/* final */static boolean AddTracker = false ;
++
++    public static BlockMgr tracker(BlockMgr blockMgr) {
++        if ( blockMgr instanceof BlockMgrTracker )
++            return blockMgr ;
++        return BlockMgrTracker.track(blockMgr) ;
++    }
++
++    /** Add a tracker if the system default is to do so */
++    private static BlockMgr track(BlockMgr blockMgr) {
++        if ( !AddTracker )
++            return blockMgr ;
++        return tracker(blockMgr) ;
++    }
++
++    public static BlockMgr create(FileSet fileSet, String ext, BlockParams params) {
++        return create(fileSet, ext,
++                      params.getFileMode(),
++                      params.getBlockSize(),
++                      params.getBlockReadCacheSize(),
++                      params.getBlockWriteCacheSize()) ;
++    }
++    
++    public static BlockMgr create(FileSet fileSet, String ext, int blockSize, int readBlockCacheSize, int writeBlockCacheSize) {
++        return create(fileSet, ext, null, blockSize, readBlockCacheSize, writeBlockCacheSize) ;
++    }
++
++    // XXX Deprecate?
++    public static BlockMgr create(FileSet fileSet, String ext, FileMode fileMode, int blockSize, int readBlockCacheSize, int writeBlockCacheSize) {
++        if ( fileSet.isMem() )
++            return createMem(fileSet.filename(ext), blockSize) ;
++        else
++            return createFile(fileSet.filename(ext), fileMode, blockSize, readBlockCacheSize, writeBlockCacheSize) ;
++    }
++
++    /** Create an in-memory block manager */
++    public static BlockMgr createMem(String indexName, int blockSize) {
++        BlockAccess file = new BlockAccessMem(indexName, blockSize) ;
++        BlockMgr blockMgr = new BlockMgrFileAccess(file, blockSize) ;
++        blockMgr = new BlockMgrFreeChain(blockMgr) ;
++        // Small cache - testing.
++        blockMgr = BlockMgrCache.create(3, 3, blockMgr) ;
++        return track(blockMgr) ;
++    }
++
++    /** Create a BlockMgr backed by a real file */
++    public static BlockMgr createFile(String filename, BlockParams params) {
++        return createFile(filename, 
++                          params.getFileMode(), params.getBlockSize(),
++                          params.getBlockReadCacheSize(), params.getBlockWriteCacheSize()) ;
++    }
++
++        /** Create a BlockMgr backed by a real file */
++    public static BlockMgr createFile(String filename, FileMode fileMode, int blockSize, int readBlockCacheSize, int writeBlockCacheSize) {
++        if ( fileMode == null )
++            fileMode = SystemIndex.fileMode() ;
++        switch (fileMode) {
++            case mapped :
++                return createMMapFile(filename, blockSize) ;
++            case direct :
++                return createStdFile(filename, blockSize, readBlockCacheSize, writeBlockCacheSize) ;
++        }
++        throw new DBOpEnvException("Unknown file mode: " + fileMode) ;
++    }
++
++    /** Create a NIO Block Manager */
++    public static BlockMgr createMMapFile(String filename, int blockSize) {
++        BlockAccess file = new BlockAccessMapped(filename, blockSize) ;
++        BlockMgr blockMgr = wrapFileAccess(file, blockSize) ;
++        return track(blockMgr) ;
++    }
++
++    /** Create a Block Manager using direct access (and a cache) */
++    public static BlockMgr createStdFile(String filename, int blockSize, int readBlockCacheSize, int writeBlockCacheSize) {
++        BlockAccess file = new BlockAccessDirect(filename, blockSize) ;
++        BlockMgr blockMgr = wrapFileAccess(file, blockSize) ;
++        blockMgr = addCache(blockMgr, readBlockCacheSize, writeBlockCacheSize) ;
++        return track(blockMgr) ;
++    }
++
++    /** Create a Block Manager using direct access, no caching, no nothing. */
++    public static BlockMgr createStdFileNoCache(String filename, int blockSize) {
++        BlockAccess blockAccess = new BlockAccessDirect(filename, blockSize) ;
++        BlockMgr blockMgr = new BlockMgrFileAccess(blockAccess, blockSize) ;
++        return blockMgr ;
++    }
++
++    /**
++     * Add a caching layer to a BlockMgr.
++     * <p>
++     * This does not make sense for memory BlockMgr or for memory mapper files.
++     * This function always add the cache.
++     * 
++     * @see #addCache(BlockMgr, FileSet, FileMode, int, int)
++     */
++    public static BlockMgr addCache(BlockMgr blockMgr, int readBlockCacheSize, int writeBlockCacheSize) {
++        if ( blockMgr instanceof BlockMgrCache )
++            Log.warn(BlockMgrFactory.class, "BlockMgr already has a cache: " + blockMgr.getLabel()) ;
++        return BlockMgrCache.create(readBlockCacheSize, writeBlockCacheSize, blockMgr) ;
++    }
++
++    /**
++     * Add a caching layer to a BlockMgr if appropriate. This does not make
++     * sense for memory BlockMgr or for memory mapper files. These are skipped.
++     */
++    public static BlockMgr addCache(BlockMgr blockMgr, FileSet fileSet, FileMode fileMode, int readBlockCacheSize, int writeBlockCacheSize) {
++        if ( fileSet.isMem() )
++            return blockMgr ;
++        if ( fileMode == null )
++            fileMode = SystemIndex.fileMode() ;
++        if ( fileMode == FileMode.mapped )
++            return blockMgr ;
++        return addCache(blockMgr, readBlockCacheSize, writeBlockCacheSize) ;
++    }
++
++    private static BlockMgr wrapFileAccess(BlockAccess blockAccess, int blockSize) {
++        BlockMgr blockMgr = new BlockMgrFileAccess(blockAccess, blockSize) ;
++        // This is a temporary fix to the problem
++        blockMgr = new BlockMgrFreeChain(blockMgr) ;
++        return blockMgr ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFileAccess.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFileAccess.java
index 0000000,0000000..c13ac2c
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFileAccess.java
@@@ -1,0 -1,0 +1,168 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block ;
++
++import static java.lang.String.format ;
++
++import org.apache.jena.atlas.lib.InternalErrorException ;
++import org.apache.jena.atlas.lib.Lib ;
++import org.seaborne.dboe.base.file.BlockAccess ;
++import org.slf4j.Logger ;
++import org.slf4j.LoggerFactory ;
++
++/**
++ * Block manager that maps from the FileAccess layer to a BlockMgr. Add free
++ * block management (but we should layer with BlockMgrFreeChain)
++ */
++
++final public class BlockMgrFileAccess extends BlockMgrBase {
++    private static Logger     log        = LoggerFactory.getLogger(BlockMgrFileAccess.class) ;
++
++    private final BlockAccess file ;
++    private boolean           closed     = false ;
++    // Set on any write operations.
++    private boolean           syncNeeded = false ;
++
++    // Create via the BlockMgrFactory.
++    /* package */BlockMgrFileAccess(BlockAccess blockAccess, int blockSize) {
++        super(blockAccess.getLabel(), blockSize) ;
++        file = blockAccess ;
++    }
++
++    @Override
++    protected Block allocate() {
++        syncNeeded = true ;
++        return file.allocate(blockSize) ;
++    }
++
++    @Override
++    public Block promote(Block block) {
++        return block ;
++    }
++
++    @Override
++    public Block getRead(long id) {
++        return getBlock(id, true) ;
++    }
++
++    @Override
++    public Block getWrite(long id) {
++        return getBlock(id, false) ;
++    }
++
++    private Block getBlock(long id, boolean readOnly) {
++        checkNotClosed() ;
++        Block block = file.read(id) ;
++        block.setReadOnly(readOnly) ;
++        return block ;
++    }
++
++    private void checkNotClosed() {
++        if ( closed )
++            throw new InternalErrorException(Lib.className(this)+" : already closed") ;
++    }
++
++    @Override
++    public void release(Block block) {
++        checkNotClosed() ;
++    }
++
++    @Override
++    public void write(Block block) {
++        checkNotClosed() ;
++        if ( block.isReadOnly() )
++            throw new BlockException("Attempt to write a read-only block ("+block.getId()+")" ) ;
++        syncNeeded = true ;
++        file.write(block) ;
++    }
++
++    @Override
++    public void overwrite(Block block) {
++        checkNotClosed() ;
++        syncNeeded = true ;
++        file.overwrite(block) ;
++    }
++
++    @Override
++    public void free(Block block) {
++        checkNotClosed() ;
++        // syncNeeded = true ;
++        // We do nothing about free blocks currently.
++    }
++
++    @Override
++    public boolean valid(int id) {
++        checkNotClosed() ;
++        return file.valid(id) ;
++    }
++
++    @Override
++    public void sync() {
++        checkNotClosed() ;
++        if ( syncNeeded )
++            file.sync() ;
++        else
++            syncNeeded = true ;
++        syncNeeded = false ;
++    }
++
++    @Override
++    public void syncForce() {
++        checkNotClosed() ;
++        file.sync() ;
++    }
++
++    @Override
++    public boolean isClosed() {
++        return closed ;
++    }
++
++    @Override
++    public void close() {
++        closed = true ;
++        file.close() ;
++    }
++
++    @Override
++    public boolean isEmpty() {
++        checkNotClosed() ;
++        return file.isEmpty() ;
++    }
++    
++    @Override
++    public long allocLimit() {
++        checkNotClosed() ;
++        return file.allocBoundary() ;
++    }
++    
++    @Override
++    public void resetAlloc(long boundary) {
++        checkNotClosed() ;
++        file.resetAllocBoundary(boundary);
++    }
++
++    @Override
++    public String toString() {
++        return format("BlockMgrFileAccess[%d bytes]:%s", blockSize, file) ;
++    }
++
++    @Override
++    protected Logger log() {
++        return log ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFreeChain.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFreeChain.java
index 0000000,0000000..b131507
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrFreeChain.java
@@@ -1,0 -1,0 +1,83 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block ;
++
++import java.util.ArrayDeque ;
++import java.util.Deque ;
++
++/**
++ * Recycle blocks - but only in-session. At the end of JVM run, the blocks are
++ * made "permanent" as no one finds them again on restart.
++ */
++final public class BlockMgrFreeChain extends BlockMgrWrapper {
++    // Could keep Pair<Integer, ByteBuffer>
++    // List<Block> freeBlocks = new ArrayList<Block>() ;
++    private final Deque<Block> freeBlocks = new ArrayDeque<>() ;    // Keep as heap?
++
++    public BlockMgrFreeChain(BlockMgr blockMgr) {
++        super(blockMgr) ;
++    }
++
++    @Override
++    public Block allocate(int blockSize) {
++        if ( !freeBlocks.isEmpty() ) {
++            Block block = freeBlocks.removeFirst() ;
++            block.getByteBuffer().position(0) ;
++            return block ;
++        }
++        return super.allocate(blockSize) ;
++    }
++
++    @Override
++    public void free(Block block) {
++        if ( block.getId() >= blockMgr.allocLimit() )
++            freeBlocks.add(block) ;
++    }
++
++    @Override
++    public void resetAlloc(long boundary) {
++        super.resetAlloc(boundary);
++        // Just clear - assumes this is effectively a transaction boundary.
++        freeBlocks.clear(); 
++//        Iterator<Block> iter = freeBlocks.iterator() ;
++//        while(iter.hasNext()) {
++//            Block blk = iter.next() ;
++//            if ( blk.getId() < boundary )
++//                iter.remove();
++//        }
++    }
++    @Override
++    public boolean valid(int id) {
++        for ( Block blk : freeBlocks ) {
++            if ( blk.getId() == id )
++                return true ;
++        }
++        return super.valid(id) ;
++    }
++
++    @Override
++    public void sync() {
++        // Flush free blocks?
++        super.sync() ;
++    }
++
++    @Override
++    public String toString() {
++        return "Free:" + super.toString() ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrLogger.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrLogger.java
index 0000000,0000000..bd218b8
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrLogger.java
@@@ -1,0 -1,0 +1,180 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block ;
++
++import org.slf4j.Logger ;
++import org.slf4j.LoggerFactory ;
++
++public class BlockMgrLogger implements BlockMgr // extends BlockMgrWrapper
++{
++    private final BlockMgr  blockMgr ;
++    protected final Logger  log ;
++    protected final boolean logAllOperations ;
++    private final String    label ;
++
++    public BlockMgrLogger(BlockMgr blockMgr, boolean logAllOperations) {
++        this(null, blockMgr.getLabel(), blockMgr, logAllOperations) ;
++    }
++
++    public BlockMgrLogger(String label, BlockMgr blockMgr, boolean logAllOperations) {
++        this(null, label, blockMgr, logAllOperations) ;
++    }
++
++    public BlockMgrLogger(Logger log, String label, BlockMgr blockMgr, boolean logAllOperations) {
++        this.blockMgr = blockMgr ;
++        if ( log == null )
++            log = LoggerFactory.getLogger(BlockMgr.class) ;
++        this.log = log ;
++        this.logAllOperations = logAllOperations ;
++        if ( label == null )
++            label = blockMgr.getLabel() ; 
++        this.label = label ;
++    }
++
++    @Override
++    public String getLabel() {
++        return label ;
++    }
++
++    @Override
++    public Block allocate(int blockSize) {
++        Block x = blockMgr.allocate(blockSize) ;
++        info("Allocate(" + x.getId() + ")") ;
++        return x ;
++    }
++
++    @Override
++    public boolean isEmpty() {
++        info("isEmpty()") ;
++        return blockMgr.isEmpty() ;
++    }
++
++    @Override
++    public long allocLimit() {
++        info("limit()") ;
++        return blockMgr.allocLimit() ;
++    }
++
++    @Override
++    public void resetAlloc(long boundary) {
++        info("resetAlloc("+boundary+")") ;
++        blockMgr.resetAlloc(boundary) ;
++    }
++    
++    @Override
++    public Block getRead(long id) {
++        info("getRead(" + id + ")") ;
++        return blockMgr.getRead(id) ;
++    }
++
++    @Override
++    public Block getWrite(long id) {
++        info("getWrite(" + id + ")") ;
++        return blockMgr.getWrite(id) ;
++    }
++
++    @Override
++    public Block promote(Block block) {
++        info("promote(" + block.getId() + ")") ;
++        return blockMgr.promote(block) ;
++    }
++
++    @Override
++    public void release(Block block) {
++        info("release(" + block.getId() + ")") ;
++        blockMgr.release(block) ;
++    }
++
++    @Override
++    public void write(Block block) {
++        info("write(" + block.getId() + ")") ;
++        blockMgr.write(block) ;
++    }
++
++    @Override
++    public void overwrite(Block block) {
++        info("overwrite(" + block.getId() + ")") ;
++        blockMgr.overwrite(block) ;
++    }
++
++    @Override
++    public void free(Block block) {
++        info("freeBlock(" + block.getId() + ")") ;
++        blockMgr.free(block) ;
++    }
++
++    @Override
++    public boolean valid(int id) {
++        info("valid(" + id + ")") ;
++        return blockMgr.valid(id) ;
++    }
++
++    @Override
++    public void close() {
++        info("close") ;
++        blockMgr.close() ;
++    }
++
++    @Override
++    public boolean isClosed() {
++        info("isClosed") ;
++        return blockMgr.isClosed() ;
++    }
++
++    @Override
++    public void sync() {
++        info("sync") ;
++        blockMgr.sync() ;
++    }
++
++    @Override
++    public void syncForce() {
++        info("syncForce") ;
++        blockMgr.syncForce() ;
++    }
++
++    @Override
++    public void beginRead() {
++        info("> start read") ;
++        blockMgr.beginRead() ;
++    }
++
++    @Override
++    public void endRead() {
++        info("< finish read") ;
++        blockMgr.endRead() ;
++    }
++
++    @Override
++    public void beginUpdate() {
++        info("> start update") ;
++        blockMgr.beginUpdate() ;
++    }
++
++    @Override
++    public void endUpdate() {
++        info("< finish update") ;
++        blockMgr.endUpdate() ;
++    }
++
++    private void info(String string) {
++        if ( label != null )
++            string = label + ": " + string ;
++        log.info(string) ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrReadonly.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrReadonly.java
index 0000000,0000000..eca86af
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrReadonly.java
@@@ -1,0 -1,0 +1,44 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block;
++
++import org.seaborne.dboe.DBOpEnvException ;
++
++public class BlockMgrReadonly extends BlockMgrWrapper
++{
++    public BlockMgrReadonly(BlockMgr blockMgr) {
++        super(blockMgr) ;
++    }
++
++    @Override public void beginUpdate()                 { throw new DBOpEnvException("Read-only block manager") ; }
++    @Override public void endUpdate()                   { throw new DBOpEnvException("Read-only block manager") ; }
++    @Override public Block allocate(int blockSize)      { throw new DBOpEnvException("Read-only block manager") ; }
++    @Override public Block getWrite(long id)            { throw new DBOpEnvException("Read-only block manager") ; }
++    @Override public Block promote(Block block)         { throw new DBOpEnvException("Read-only block manager") ; }
++    @Override public void write(Block block)            { throw new DBOpEnvException("Read-only block manager") ; }
++    @Override public void overwrite(Block block)        { throw new DBOpEnvException("Read-only block manager") ; }
++    @Override public void free(Block block)             { throw new DBOpEnvException("Read-only block manager") ; }
++    
++//    @Override 
++//    public void sync()
++//    {
++//        blockMgr.sync() ;
++//    }
++
++    @Override public String toString()          { return "RO:"+super.toString() ; } 
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrSwitcher.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrSwitcher.java
index 0000000,0000000..1a05120
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrSwitcher.java
@@@ -1,0 -1,0 +1,39 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block;
++
++public class BlockMgrSwitcher extends BlockMgrWrapper 
++{
++    protected final BlockMgr blockMgr1 ;
++    protected final BlockMgr blockMgr2 ;
++
++    public BlockMgrSwitcher(BlockMgr blockMgr1, BlockMgr blockMgr2)
++    {
++        super(blockMgr1) ;
++        this.blockMgr1 = blockMgr1 ;
++        this.blockMgr2 = blockMgr2 ;
++    }
++
++    public void switchover()
++    {
++        if ( super.blockMgr == blockMgr1 )
++            setBlockMgr(blockMgr2) ;
++        else
++            setBlockMgr(blockMgr1) ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrSync.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrSync.java
index 0000000,0000000..492f543
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrSync.java
@@@ -1,0 -1,0 +1,148 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block ;
++
++
++/**
++ * Add synchronized to a BlockMgr. This is the same as BlockMgrWrapper but with
++ * 'synchronized' added
++ */
++
++public class BlockMgrSync implements BlockMgr {
++    protected final BlockMgr blockMgr ;
++
++    public BlockMgrSync(BlockMgr blockMgr) {
++        this.blockMgr = blockMgr ;
++    }
++
++    public BlockMgr getWrapped() {
++        return blockMgr ;
++    }
++
++    @Override
++    synchronized public Block allocate(int blockSize) {
++        return blockMgr.allocate(blockSize) ;
++    }
++
++    @Override
++    synchronized public Block getRead(long id) {
++        return blockMgr.getRead(id) ;
++    }
++
++    @Override
++    synchronized public Block getWrite(long id) {
++        return blockMgr.getWrite(id) ;
++    }
++
++    @Override
++    synchronized public Block promote(Block block) {
++        return blockMgr.promote(block) ;
++    }
++
++    @Override
++    synchronized public void release(Block block) {
++        blockMgr.release(block) ;
++    }
++
++    @Override
++    synchronized public void write(Block block) {
++        blockMgr.write(block) ;
++    }
++
++    @Override
++    synchronized public void overwrite(Block block) {
++        blockMgr.overwrite(block) ;
++    }
++
++    @Override
++    synchronized public void free(Block block) {
++        blockMgr.free(block) ;
++    }
++
++    @Override
++    synchronized public void sync() {
++        blockMgr.sync() ;
++    }
++
++    @Override
++    synchronized public void syncForce() {
++        blockMgr.syncForce() ;
++    }
++
++    @Override
++    synchronized public void close() {
++        blockMgr.close() ;
++    }
++
++    @Override
++    synchronized public boolean isEmpty() {
++        return blockMgr.isEmpty() ;
++    }
++
++    @Override
++    synchronized public long allocLimit() {
++        return blockMgr.allocLimit() ;
++    }
++
++    
++    @Override
++    synchronized public void resetAlloc(long boundary) {
++        blockMgr.resetAlloc(boundary) ;
++    }
++    
++    @Override
++    synchronized public void beginRead() {
++        blockMgr.beginRead() ;
++    }
++
++    @Override
++    synchronized public void endRead() {
++        blockMgr.endRead() ;
++    }
++
++    @Override
++    synchronized public void beginUpdate() {
++        blockMgr.beginUpdate() ;
++    }
++
++    @Override
++    synchronized public void endUpdate() {
++        blockMgr.endUpdate() ;
++    }
++
++    @Override
++    synchronized public boolean valid(int id) {
++        return blockMgr.valid(id) ;
++    }
++
++    @Override
++    synchronized public boolean isClosed() {
++        return blockMgr.isClosed() ;
++    }
++
++    @Override
++    synchronized public String getLabel() {
++        return blockMgr.getLabel() ;
++    }
++
++    @Override
++    public String toString() {
++        return "Sync:" + blockMgr.toString() ;
++    }
++
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrTracker.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrTracker.java
index 0000000,0000000..588a455
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrTracker.java
@@@ -1,0 -1,0 +1,413 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block ;
++
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.Alloc ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.BeginRead ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.BeginUpdate ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.EndRead ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.EndUpdate ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.Free ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.GetRead ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.GetWrite ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.Promote ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.Release ;
++import static org.seaborne.dboe.base.block.BlockMgrTracker.Action.Write ;
++
++import java.util.ArrayList ;
++import java.util.List ;
++
++import org.apache.jena.atlas.lib.Pair ;
++import org.apache.jena.ext.com.google.common.collect.HashMultiset ;
++import org.apache.jena.ext.com.google.common.collect.Multiset ;
++import org.seaborne.dboe.DBOpEnvException ;
++import org.slf4j.Logger ;
++import org.slf4j.LoggerFactory ;
++
++public class BlockMgrTracker /* extends BlockMgrWrapper */ implements BlockMgr {
++    private static Logger loggerDefault = LoggerFactory.getLogger(BlockMgrTracker.class) ;
++    public static boolean verbose       = false ;
++
++    static enum Action {
++        Alloc, Promote, GetRead, GetWrite, Write, Release, Free, 
++        BeginRead, EndRead, 
++        BeginUpdate, EndUpdate
++    }
++    static final Long                        NoId              = (long)-9 ;
++
++    // ---- State for tracking
++    // Track and count block references and releases
++    // No - the page is dirty.
++    
++    
++    protected final Multiset<Long>           activeReadBlocks  = HashMultiset.create() ;
++    protected final Multiset<Long>           activeWriteBlocks = HashMultiset.create() ;
++    // Track the operations
++    protected final List<Pair<Action, Long>> actions           = new ArrayList<>() ;
++    // ---- State for tracking
++
++    protected final BlockMgr                 blockMgr ;
++
++    private void clearInternalRW() {
++        activeReadBlocks.clear() ;
++        activeWriteBlocks.clear() ;
++        actions.clear() ;
++    }
++
++    private void clearInternalIter() {
++        clearInternalRW() ;
++    }
++
++    private int          inRead     = 0 ;
++    private boolean      inUpdate   = false ;
++    private boolean      inBatch    = false ;
++    private final Logger log ;
++    private final String label ;
++
++    public static BlockMgr track(BlockMgr blkMgr) {
++        return track(blkMgr.getLabel(), blkMgr) ;
++    }
++
++    private static BlockMgr track(String label, BlockMgr blkMgr) {
++        return new BlockMgrTracker(label, blkMgr) ;
++    }
++
++    private BlockMgrTracker(BlockMgr blockMgr) {
++        this(LoggerFactory.getLogger(BlockMgrTracker.class), blockMgr.getLabel(), blockMgr) ;
++    }
++
++    private BlockMgrTracker(String label, BlockMgr blockMgr) {
++        this(loggerDefault, label, blockMgr) ;
++    }
++
++    //
++    // public BlockMgrTracker(Class<?> cls, String label, BlockMgr blockMgr)
++    // {
++    // this(LoggerFactory.getLogger(cls), label, blockMgr) ;
++    // }
++
++    private BlockMgrTracker(Logger logger, String label, BlockMgr blockMgr) {
++        this.blockMgr = blockMgr ;
++        this.log = logger ;
++        this.label = blockMgr.getLabel() ;
++    }
++
++    private void add(Action action, Long id) {
++        actions.add(new Pair<>(action, id)) ;
++    }
++
++    @Override
++    public Block allocate(int blockSize) {
++        Block block ;
++        synchronized (this) {
++            checkUpdate(Alloc) ;
++            block = blockMgr.allocate(blockSize) ;
++            Long id = block.getId() ;
++            activeWriteBlocks.add(id) ;
++            add(Alloc, id) ;
++        }
++        return block ;
++    }
++
++    @Override
++    public Block getRead(long id) {
++        // What if this is a write block already?
++        synchronized (this) {
++            checkRead(GetRead) ;
++            Long x = id ;
++            add(GetRead, x) ;
++
++            if ( activeWriteBlocks.contains(x) )
++                activeWriteBlocks.add(x) ;
++            else
++                activeReadBlocks.add(x) ;
++        }
++        return blockMgr.getRead(id) ;
++    }
++
++    @Override
++    public Block getWrite(long id) {
++        synchronized (this) {
++            checkUpdate(GetWrite) ;
++            Long x = id ;
++            add(GetWrite, x) ;
++            activeWriteBlocks.add(x) ;
++        }
++        return blockMgr.getWrite(id) ;
++    }
++
++    @Override
++    public Block promote(Block block) {
++        synchronized (this) {
++            checkUpdate(Promote) ;
++            Long id = block.getId() ;
++            add(Promote, id) ;
++
++            if ( !activeWriteBlocks.contains(id) && !activeReadBlocks.contains(id) )
++                error(Promote, id + " is not an active block") ;
++
++            if ( activeReadBlocks.contains(id) )
++                // Remove one read count
++                // Really, do this if obtained via "getRead" and not "getWrite"
++                activeReadBlocks.remove(id) ;
++
++            // Double promotion results in only one entry.
++            if ( !activeWriteBlocks.contains(id) )
++                activeWriteBlocks.add(id) ;
++        }
++        return blockMgr.promote(block) ;
++    }
++
++    @Override
++    public void release(Block block) {
++        synchronized (this) {
++            checkRead(Release) ;
++            Long id = block.getId() ;
++            add(Release, id) ;
++
++            if ( !activeReadBlocks.contains(id) && !activeWriteBlocks.contains(id) )
++                error(Release, id + " is not an active block") ;
++
++            // May have been promoted.
++            if ( activeWriteBlocks.contains(id) )
++                activeWriteBlocks.remove(id) ;
++            else
++                activeReadBlocks.remove(block.getId()) ;
++        }
++        blockMgr.release(block) ;
++    }
++
++    @Override
++    public void write(Block block) {
++        writeTracker(block) ;
++        blockMgr.write(block) ;
++    }
++
++    @Override
++    synchronized public void overwrite(Block block) {
++        writeTracker(block) ;
++        blockMgr.overwrite(block) ;
++    }
++
++    private void writeTracker(Block block) {
++        synchronized (this) {
++            checkUpdate(Write) ;
++            Long id = block.getId() ;
++            add(Write, id) ;
++            if ( !activeWriteBlocks.contains(id) )
++                error(Write, id + " is not an active write block") ;
++        }
++    }
++
++    @Override
++    public void free(Block block) {
++        synchronized (this) {
++            checkUpdate(Free) ;
++            Long id = block.getId() ;
++            add(Free, id) ;
++            if ( activeReadBlocks.contains(id) )
++                error(Free, id + " is a read block") ;
++            else if ( !activeWriteBlocks.contains(id) )
++                error(Free, id + " is not a write block") ;
++            activeWriteBlocks.remove(id) ;
++        }
++        blockMgr.free(block) ;
++    }
++
++    @Override
++    public void sync() {
++        blockMgr.sync() ;
++    }
++
++    @Override
++    public void syncForce() {
++        blockMgr.syncForce() ;
++    }
++
++    @Override
++    public void close() {
++        blockMgr.close() ;
++    }
++
++    @Override
++    public boolean isEmpty() {
++        return blockMgr.isEmpty() ;
++    }
++    
++    @Override
++    public long allocLimit() {
++        return blockMgr.allocLimit() ;
++    }
++    
++    @Override
++    public void resetAlloc(long boundary) {
++        blockMgr.resetAlloc(boundary) ;
++    }
++    
++    @Override
++    public boolean valid(int id) {
++        return blockMgr.valid(id) ;
++    }
++
++    @Override
++    public boolean isClosed() {
++        return blockMgr.isClosed() ;
++    }
++
++    @Override
++    synchronized public void beginRead() {
++        synchronized (this) {
++            if ( inUpdate )
++                error(BeginRead, "beginRead when already in update") ;
++            inRead++ ;
++            inUpdate = false ;
++        }
++        blockMgr.beginRead() ;
++    }
++
++    @Override
++    synchronized public void endRead() {
++        synchronized (this) {
++            if ( inRead == 0 )
++                error(EndRead, "endRead but not in read") ;
++            if ( inUpdate )
++                error(EndRead, "endRead when in update") ;
++
++            checkEmpty("Outstanding write blocks at end of read operations!", activeWriteBlocks) ;
++
++            if ( inRead == 0 ) {
++                // Check at end of multiple reads or a write
++                checkEmpty("Outstanding read blocks at end of read operations", activeReadBlocks) ;
++                clearInternalRW() ;
++            }
++
++            inUpdate = false ;
++            inRead-- ;
++        }
++        blockMgr.endRead() ;
++    }
++
++    @Override
++    public void beginUpdate() {
++        synchronized (this) {
++            if ( inRead > 0 )
++                error(BeginUpdate, "beginUpdate when already in read") ;
++            if ( inUpdate )
++                error(BeginUpdate, "beginUpdate when already in update") ;
++            inUpdate = true ;
++        }
++        blockMgr.beginUpdate() ;
++    }
++
++    @Override
++    public void endUpdate() {
++        synchronized (this) {
++            if ( !inUpdate )
++                error(EndUpdate, "endUpdate but not in update") ;
++            if ( inRead > 0 )
++                error(EndUpdate, "endUpdate when in read") ;
++
++            checkEmpty("Outstanding read blocks at end of update operations", activeReadBlocks) ;
++
++            checkEmpty("Outstanding write blocks at end of update operations", activeWriteBlocks) ;
++
++            inUpdate = false ;
++            inRead = 0 ;
++            clearInternalRW() ;
++        }
++        blockMgr.endUpdate() ;
++    }
++
++    private void checkUpdate(Action action) {
++//        if ( ! inBatch )
++//            warn(action, "update called outside a batch") ;
++        if ( ! inUpdate )
++            error(action, "called outside update") ;
++    }
++
++    private void checkRead(Action action) {
++        if ( !inUpdate && inRead == 0 )
++            error(action, "Called outside update and read") ;
++    }
++
++    private void checkEmpty(String string, Multiset<Long> blocks) {
++        if ( !blocks.isEmpty() ) {
++            error(string) ;
++            for ( Long id : blocks )
++                info("    Block: " + id) ;
++            history() ;
++            throw new DBOpEnvException() ;
++            // debugPoint() ;
++        }
++    }
++
++    private String msg(String string) {
++        if ( label == null )
++            return string ;
++        return label + ": " + string ;
++    }
++
++    private void info(String string) {
++        log.info(msg(string)) ;
++    }
++
++    private void warn(String string) {
++        log.warn(msg(string)) ;
++    }
++
++    private void warn(Action action, String string) {
++        warn(action + ": " + string) ;
++    }
++
++    private void error(String string) {
++        log.error(msg(string)) ;
++    }
++
++    private void error(Action action, String string) {
++        if ( verbose ) {
++            error(action + ": " + string) ;
++            history() ;
++        }
++        throw new BlockException(msg(action + ": " + string)) ;
++        // debugPoint() ;
++    }
++
++    // Do nothing - but use a a breakpoint point.
++    private void debugPoint() {}
++
++    private void history() {
++        info("History") ;
++        for ( Pair<Action, Long> p : actions ) {
++            if ( p.getRight() != NoId )
++                log.info(String.format("%s:     %-12s  %d", label, p.getLeft(), p.getRight())) ;
++            else
++                log.info(String.format("%s:     %-12s", label, p.getLeft())) ;
++
++        }
++    }
++
++    @Override
++    public String toString() {
++        return "BlockMgrTracker" + ((label == null) ? "" : (": " + label)) ;
++    }
++
++    @Override
++    public String getLabel() {
++        return label ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrWrapper.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrWrapper.java
index 0000000,0000000..029d309
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockMgrWrapper.java
@@@ -1,0 -1,0 +1,148 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block ;
++
++
++public class BlockMgrWrapper implements BlockMgr {
++    protected BlockMgr blockMgr ;
++
++    public BlockMgrWrapper(BlockMgr blockMgr) {
++        setBlockMgr(blockMgr) ;
++    }
++
++    /** Set another BlockMgr as the target of the wrapper - return the old one */
++    protected final BlockMgr setBlockMgr(BlockMgr blockMgr) {
++        BlockMgr old = this.blockMgr ;
++        this.blockMgr = blockMgr ;
++        return old ;
++    }
++
++    public BlockMgr getWrapped() {
++        return blockMgr ;
++    }
++
++    @Override
++    public Block allocate(int blockSize) {
++        return blockMgr.allocate(blockSize) ;
++    }
++
++    @Override
++    public Block getRead(long id) {
++        return blockMgr.getRead(id) ;
++    }
++
++    @Override
++    public Block getWrite(long id) {
++        return blockMgr.getWrite(id) ;
++    }
++
++    @Override
++    public Block promote(Block block) {
++        return blockMgr.promote(block) ;
++    }
++
++    @Override
++    public void release(Block block) {
++        blockMgr.release(block) ;
++    }
++
++    @Override
++    public void write(Block block) {
++        blockMgr.write(block) ;
++    }
++
++    @Override
++    public void overwrite(Block block) {
++        blockMgr.overwrite(block) ;
++    }
++
++    @Override
++    public void free(Block block) {
++        blockMgr.free(block) ;
++    }
++
++    @Override
++    public boolean isEmpty() {
++        return blockMgr.isEmpty() ;
++    }
++    
++    @Override
++    public long allocLimit() {
++        return blockMgr.allocLimit() ;
++    }
++    
++    @Override
++    public void resetAlloc(long boundary) {
++        blockMgr.resetAlloc(boundary) ;
++    }
++    
++    @Override
++    public void sync() {
++        blockMgr.sync() ;
++    }
++
++    @Override
++    public void syncForce() {
++        blockMgr.syncForce() ;
++    }
++
++    @Override
++    public boolean valid(int id) {
++        return blockMgr.valid(id) ;
++    }
++
++    @Override
++    public boolean isClosed() {
++        return blockMgr.isClosed() ;
++    }
++
++    @Override
++    public void close() {
++        blockMgr.close() ;
++    }
++
++    @Override
++    public void beginRead() {
++        blockMgr.beginRead() ;
++    }
++
++    @Override
++    public void endRead() {
++        blockMgr.endRead() ;
++    }
++
++    @Override
++    public void beginUpdate() {
++        blockMgr.beginUpdate() ;
++    }
++
++    @Override
++    public void endUpdate() {
++        blockMgr.endUpdate() ;
++    }
++
++    @Override
++    public String getLabel() {
++        return blockMgr.getLabel() ;
++    }
++
++    @Override
++    public String toString() {
++        return blockMgr.toString() ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockParams.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockParams.java
index 0000000,0000000..28afbfa
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockParams.java
@@@ -1,0 -1,0 +1,26 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block;
++
++public interface BlockParams {
++    public FileMode getFileMode() ;
++    public Integer getBlockSize() ;
++    public Integer getBlockReadCacheSize() ;
++    public Integer getBlockWriteCacheSize() ;
++}
++

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockType.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockType.java
index 0000000,0000000..0acc6b1
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/BlockType.java
@@@ -1,0 -1,0 +1,68 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block;
++
++import org.apache.jena.atlas.io.IndentedWriter ;
++import org.apache.jena.atlas.io.Printable ;
++import org.seaborne.dboe.DBOpEnvException ;
++
++import org.apache.jena.sparql.util.Named ;
++
++public enum BlockType implements Printable, Named
++{
++    // The id should fit into an unsigned byte.
++    FREE(-1, "Free"), 
++    BTREE_NODE(5, "BTreeNode") ,
++    BPTREE_BRANCH(6, "BPlusTreeBranch") ,
++    BPTREE_LEAF(7, "BPlusTreeLeaf") ,
++    DICTIONARY(10, "Dictionary") ,
++    RECORD_BLOCK(99, "RecordBlock"),
++    UNDEF(255, "UndefinedBlockType")
++    ;
++
++    private final int id ;
++    private final String name ;
++
++    BlockType(int id, String name)
++    {
++        this.id = id ;
++        this.name = name ;
++        
++    }
++    
++    @Override
++    public void output(IndentedWriter out)
++    { out.print(getName()) ; }
++
++    final public int id() { return id ; }
++    
++    @Override
++    final public String getName() { return name ; }
++    
++    @Override public String toString() { return getName() ; }
++    
++    public static BlockType extract(int x)
++    {
++        if ( x == BTREE_NODE.id() )         return BTREE_NODE ;
++        if ( x == BPTREE_BRANCH.id() )      return BPTREE_BRANCH ;
++        if ( x == BPTREE_LEAF.id() )        return BPTREE_LEAF ;
++        if ( x == RECORD_BLOCK.id() )       return RECORD_BLOCK ;
++        if ( x == DICTIONARY.id() )         return DICTIONARY ;
++        throw new DBOpEnvException("No known block type for "+x) ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/FileMode.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/FileMode.java
index 0000000,0000000..c6531a6
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/block/FileMode.java
@@@ -1,0 -1,0 +1,26 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.block;
++
++public enum FileMode
++{
++    /** Use memory mapped files */
++    mapped,
++    /** Use in-JVM caching */
++    direct ;
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/BufferBase.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/BufferBase.java
index 0000000,0000000..ca170a4
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/BufferBase.java
@@@ -1,0 -1,0 +1,262 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.buffer;
++
++import static java.lang.String.format ;
++import static org.seaborne.dboe.sys.SystemIndex.FillByte ;
++import static org.seaborne.dboe.sys.Sys.NetworkOrder ;
++import static org.seaborne.dboe.sys.SystemIndex.NullOut ;
++
++import java.nio.ByteBuffer ;
++
++import org.apache.jena.atlas.lib.ByteBufferLib ;
++import org.seaborne.dboe.base.record.RecordException ;
++
++
++/** ByteBuffer specialization - Common operations for RecordBuffer and PtrBuffer */  
++abstract class BufferBase
++{
++    public static final boolean CheckBuffer = false ;
++    
++    protected ByteBuffer bb ;
++    protected int slotLen ;
++    protected int numSlot ;                 // Number of records in use
++    protected final int maxSlot ;           // Maximum number of records
++
++    protected BufferBase(ByteBuffer bb, int slotLen, int num)
++    {
++        if ( CheckBuffer )
++        {
++            if ( ! bb.order().equals(NetworkOrder) || !bb.hasArray() )
++                throw new RecordException("Duff buffer (byte order is not network order)") ;
++            if ( bb.limit() == 0 )
++                throw new RecordException("Duff buffer (zero length byte buffer)") ;
++            int size = bb.limit() ;
++            int slots = size/slotLen  ;
++            if ( size%slotLen != 0 )
++                throw new RecordException(format("Misaligned buffer: size=%d, keyLen=%d",size, slotLen)) ; 
++            if ( slots < num )
++                throw new RecordException(format("Wrong size: slots=%d, len=%d", slots, num)) ;
++        }
++        this.bb = bb ; 
++        this.slotLen = slotLen ;
++        this.numSlot = num ;
++        this.maxSlot = bb.limit()/slotLen ;
++        if ( ! bb.isReadOnly() && NullOut )
++            clear(numSlot, maxSlot-numSlot) ;
++    }
++    
++    // Why so many final methods?  This code is performace critical and "final" methods
++    // can be inlined by the JIT - or just not have object dispatch done each time
++    // because the destination is fixed.
++    // (and here - there are quite a few assumed relationships of the ints). 
++    
++    final
++    public void copy(int srcIdx, BufferBase dst, int dstIdx, int len)
++    {
++        if ( len == 0 )
++            return ;
++        
++        // Check end index is inside the buffer.
++        checkBounds(srcIdx+len-1, maxSlot) ;
++     
++        BufferBase src = this ;   // Clarity
++
++        if ( dst.numSlot < dstIdx-1 )
++            // Allow copy to be just off the end of dst.
++            throw new IllegalArgumentException(format("copy: Out of bounds: dstIdx=%d, dst size=%d", dstIdx, dst.numSlot)) ;
++        if ( src.slotLen != dst.slotLen )
++            throw new RecordException(format("copy: records of differnt sizes: %d, %d",src.slotLen, dst.slotLen)) ;
++        
++        // How do we set the numRec in dst? max(dstIdx+len, old count)  
++        ByteBufferLib.bbcopy(src.bb, srcIdx, dst.bb, dstIdx, len, slotLen) ;
++        dst.numSlot = Math.max(dstIdx+len, dst.numSlot) ; 
++    }
++    
++    final
++    public void copyToTop(BufferBase dst)
++    { 
++        copy(0, dst, dst.numSlot, numSlot) ; } 
++    
++    /** Remove top element */
++    final public void removeTop()
++    {
++        if ( numSlot == 0 )
++            throw new IndexOutOfBoundsException("removeTop: empty buffer") ; 
++        clear(numSlot-1) ;
++        numSlot-- ;
++    }
++    
++    final public void remove(int idx) { shiftDown(idx) ; }
++    
++    /** Does not reset the size */
++    final
++    public void clear(int idx, int len)
++    {
++        if ( NullOut )
++            ByteBufferLib.bbfill(bb, idx, (idx+len), FillByte, slotLen) ;
++    }
++
++    /** Does not reset the size */
++    final public void clear()           { clear(0, maxSlot) ; numSlot = 0 ; }
++    
++    /** Does not reset the size */
++    final public void clear(int idx)    { clear(idx, 1) ; }
++    
++    /** Is the record at idx set clear or not?
++        This is done without regard to buffer size.
++        Requires NullOut to be accurate.
++        Testing.
++     */
++    final public boolean isClear(int idx)
++    {
++        checkBounds(idx, maxSlot) ;
++        int x = idx*slotLen ;
++        int y = (idx+1)*slotLen ;
++        for ( int i = x ; i < y ; i++ )
++            if ( bb.get(i) != FillByte )
++            {
++//                byte b = bb.get(i) ;
++//                lib.ByteBufferLib.print(bb) ;
++                return false ;
++            }
++        return true ;
++    }
++
++    final public boolean isFull()
++    {
++        return numSlot >= maxSlot ; 
++    }
++
++    final public boolean isEmpty()
++    {
++        return numSlot == 0 ; 
++    }
++    
++    final public void incSize() { incSize(1) ; }
++    final public void incSize(int n)
++    { 
++        if ( numSlot+n > maxSlot )
++            throw new IllegalArgumentException(format("inc(%d): out of range: max=%d", n, maxSlot)) ;
++        numSlot += n ;
++    }
++
++    final public void decSize() { decSize(1) ; }
++    final public void decSize(int n)
++    { 
++        if ( numSlot-n < 0 )
++            throw new IllegalArgumentException(format("dec(%d): out of range: max=%d", n, maxSlot)) ;
++        numSlot -= n ;
++    }
++    
++    final public int slotLen()      { return slotLen ; }
++    
++    final public int getSize()      { return numSlot ; } 
++    
++    final public void setSize(int n)
++    {
++        if ( n < 0 || n > maxSlot )
++            throw new IllegalArgumentException(format("size(%d): out of range: max=%d", n, maxSlot)) ;
++        numSlot = n ;
++    }
++    
++    final public int size()       { return numSlot ; }
++    final public int maxSize()    { return maxSlot ; }
++    
++    final public void shiftUp(int idx) { shiftUpN(idx, 1) ; }
++    final public void shiftUpN(int idx, int num)
++    {
++        checkBounds(idx, numSlot) ;
++        if ( numSlot + num > maxSlot )
++            throw new IllegalArgumentException(format("Shift up(%d): out of range: len=%d max=%d", num, num, maxSlot)) ;
++        
++        ByteBufferLib.bbcopy(bb, idx, idx+num, (numSlot-idx), slotLen) ;       // src, dst 
++        if ( NullOut )
++            clear(idx, num) ;
++
++        numSlot += num ;
++    }
++    
++    final public void shiftDown(int idx) { shiftDownN(idx, 1) ; }
++    final public void shiftDownN(int idx, int num)
++    {
++        checkBounds(idx, numSlot) ;
++        if ( idx+num > numSlot )
++            throw new IllegalArgumentException(format("Shift down(%d,%d): out of range: len=%d", idx, num, num)) ;
++
++        ByteBufferLib.bbcopy(bb, idx+num, idx, (numSlot-num-idx), slotLen) ;       // src, dst
++
++        if ( NullOut )
++            clear(numSlot-num, num) ;
++        numSlot -= num ;
++    }
++    
++//    @Override
++//    final public String toString()
++//    {
++//        StringBuilder str = new StringBuilder() ;
++//        str.append(format("Len=%d Count=%d ", bb.limit()/recLen, num)) ;
++//        
++//        for ( int i = 0 ; i < max*recLen ; i++ )
++//        {
++//            if ( i != 0 && i%recLen == 0 )
++//                str.append(" ") ;
++//            byte b = bb.get(i) ;
++//            str.append(format("%02x", b)) ;
++//        }
++//        return str.toString() ;
++//    }
++
++    /** Move the element from the high end of this to the low end of other */
++    public void shiftRight(BufferBase other)
++    {
++        if ( other.numSlot >= other.maxSlot )
++            throw new BufferException("No space in destination buffer") ;
++        if ( numSlot <= 0 )
++            throw new BufferException("Empty buffer") ;
++
++        if ( other.numSlot > 0 )
++            other.shiftUp(0) ;
++        else
++            other.numSlot++ ;
++        // Copy high to low slot.
++        ByteBufferLib.bbcopy(bb, (numSlot-1), other.bb, 0, 1, slotLen) ;
++        removeTop() ;
++    }
++
++    /** Move the element from the low end of other to the high end of this */
++    public void shiftLeft(BufferBase other)
++    {
++        if ( numSlot >= maxSlot )
++            throw new BufferException("No space in destination buffer") ;
++        if ( other.numSlot <= 0 )
++            throw new BufferException("Empty buffer") ;
++
++        // Copy low to above high slot.
++        ByteBufferLib.bbcopy(other.bb, 0, bb, numSlot, 1, slotLen) ;
++        // Correct length.
++        numSlot ++ ;
++        other.shiftDown(0) ;
++    }
++
++    final private static void checkBounds(int idx, int len)
++    {
++        if ( idx < 0 || idx >= len )
++            throw new BufferException(format("Out of bounds: idx=%d, size=%d", idx, len)) ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/BufferException.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/BufferException.java
index 0000000,0000000..ee90872
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/BufferException.java
@@@ -1,0 -1,0 +1,26 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.buffer;
++
++public class BufferException extends RuntimeException
++{
++    public BufferException()                            { super() ; }
++    public BufferException(String msg)                  { super(msg) ; }
++    public BufferException(Throwable th)                { super(th) ; }
++    public BufferException(String msg, Throwable th)    { super(msg, th) ; }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/PtrBuffer.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/PtrBuffer.java
index 0000000,0000000..0c084fa
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/PtrBuffer.java
@@@ -1,0 -1,0 +1,124 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.buffer ;
++
++import static java.lang.String.format ;
++
++import java.nio.ByteBuffer ;
++import java.nio.IntBuffer ;
++
++import org.seaborne.dboe.base.record.RecordException ;
++import org.seaborne.dboe.sys.Sys;
++import org.seaborne.dboe.sys.SystemIndex ;
++
++/** An IntBuffer with extra operations */
++
++final public class PtrBuffer extends BufferBase {
++    private IntBuffer iBuff ;
++
++    private PtrBuffer(int maxRec) {
++        this(ByteBuffer.allocate(SystemIndex.SizeOfPointer * maxRec), 0) ;
++    }
++
++    public PtrBuffer(ByteBuffer bb, int num) {
++        super(bb, SystemIndex.SizeOfPointer, num) ;
++        iBuff = bb.asIntBuffer() ;
++
++        if ( CheckBuffer ) {
++            // It is a IntBuffer with associated ByteBuffer
++            if ( iBuff.position() != 0 || bb.order() != Sys.NetworkOrder )
++                throw new RecordException("Duff pointer buffer") ;
++        }
++    }
++
++    public int get(int idx) {
++        checkBounds(idx, numSlot) ;
++        return _get(idx) ;
++    }
++
++    public int getHigh() {
++        if ( numSlot == 0 )
++            throw new IllegalArgumentException("Empty PtrBuffer") ;
++        return _get(numSlot - 1) ;
++    }
++
++    public int getLow() {
++        if ( numSlot == 0 )
++            throw new IllegalArgumentException("Empty PtrBuffer") ;
++        return _get(0) ;
++    }
++
++    public void add(int val) {
++        add(numSlot, val) ;
++    }
++
++    public void add(int idx, int val) {
++        if ( idx != numSlot ) {
++            checkBounds(idx, numSlot) ;
++            shiftUp(idx) ;
++        } else {
++            if ( numSlot >= maxSlot )
++                throw new BufferException(format("Out of bounds: idx=%d, ptrs=%d", idx, maxSlot)) ;
++            numSlot++ ;
++        }
++        // Add right at the top.
++        _set(idx, val) ;
++    }
++
++    public void set(int idx, int val) {
++        checkBounds(idx, numSlot) ;
++        _set(idx, val) ;
++    }
++
++    private final int _get(int idx) {
++        return iBuff.get(idx) ;
++    }
++
++    private final void _set(int idx, int val) {
++        iBuff.put(idx, val) ;
++    }
++
++    @Override
++    public String toString() {
++        StringBuilder str = new StringBuilder() ;
++        str.append(format("Len=%d Max=%d ", numSlot, maxSlot)) ;
++
++        for ( int i = 0 ; i < numSlot ; i++ ) {
++            if ( i != 0 )
++                str.append(" ") ;
++            int x = _get(i) ;
++            str.append(format("%04d", x)) ;
++        }
++        return str.toString() ;
++    }
++
++    private static void checkBounds(int idx, int len) {
++        if ( idx < 0 || idx >= len )
++            throw new BufferException(format("Out of bounds: idx=%d, ptrs=%d", idx, len)) ;
++    }
++
++    /**
++     * A duplicate which does not share anything with the original - for testing
++     */
++    public PtrBuffer duplicate() {
++        PtrBuffer n = new PtrBuffer(maxSlot) ;
++        copy(0, n, 0, maxSlot) ; // numSlot
++        n.numSlot = numSlot ; // Reset
++        return n ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/RecordBuffer.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/RecordBuffer.java
index 0000000,0000000..2a715bf
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/RecordBuffer.java
@@@ -1,0 -1,0 +1,235 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.buffer ;
++
++import static java.lang.String.format ;
++import static org.apache.jena.atlas.lib.Alg.encodeIndex ;
++
++import java.nio.ByteBuffer ;
++import java.util.Iterator ;
++
++import org.seaborne.dboe.base.record.Record ;
++import org.seaborne.dboe.base.record.RecordFactory ;
++import org.seaborne.dboe.base.record.RecordMapper ;
++
++final public class RecordBuffer extends BufferBase {
++    private RecordFactory factory ;
++
++    // Need own specialized binary search :-(
++
++    public RecordBuffer(RecordFactory recFactory, int maxRec) {
++        this(ByteBuffer.allocate(recFactory.recordLength() * maxRec), recFactory, 0) ;
++    }
++
++    public RecordBuffer(ByteBuffer bb, RecordFactory recFactory, int num) {
++        super(bb, recFactory.recordLength(), num) ;
++        this.factory = recFactory ;
++    }
++
++    public <X> X access(int idx, byte[] key, RecordMapper<X> mapper) {
++        checkBounds(idx, numSlot) ;
++        return _access(idx, key, mapper) ;
++    }
++
++    public Record get(int idx) {
++        checkBounds(idx, numSlot) ;
++        return _get(idx) ;
++    }
++
++    public Record getLow() {
++        if ( numSlot == 0 )
++            throw new IllegalArgumentException("getLow: Empty RecordBuffer") ;
++        return _get(0) ;
++    }
++
++    public Record getHigh() {
++        if ( numSlot == 0 )
++            throw new IllegalArgumentException("getHigh: Empty RecordBuffer") ;
++        return _get(numSlot - 1) ;
++    }
++
++    // Inserts at top.
++    public void add(Record record) {
++        add(numSlot, record) ;
++    }
++
++    // Inserts at slot idx
++    public void add(int idx, Record record) {
++        if ( idx != numSlot ) {
++            checkBounds(idx, numSlot) ;
++            shiftUp(idx) ; // Changes count.
++        } else {
++            if ( numSlot >= maxSlot )
++                throw new BufferException(format("Out of bounds: idx=%d, ptrs=%d", idx, maxSlot)) ;
++            numSlot++ ;
++        }
++        _set(idx, record) ;
++    }
++
++    // Overwrites the contents of slot idx
++    public void set(int idx, Record record) {
++        if ( idx == numSlot ) {
++            add(idx, record) ;
++            return ;
++        } else
++            checkBounds(idx, numSlot) ;
++        _set(idx, record) ;
++    }
++
++    // No checking bound : careful use only!
++
++    public <X> X _access(int idx, byte[] key, RecordMapper<X> mapper) {
++        return factory.access(bb, idx, key, mapper) ;
++    }
++
++    public Record _get(int idx) {
++        return factory.buildFrom(bb, idx) ;
++    }
++
++    // No bounds checking : careful use only!
++    void _set(int idx, Record rec) {
++        factory.insertInto(rec, bb, idx) ;
++    }
++
++    // Linear search for testing.
++    int find1(byte[] data) {
++        for ( int i = 0 ; i < numSlot ; i++ ) {
++            int x = compare(i, data) ;
++            if ( x == 0 )
++                return i ;
++            if ( x > 0 )
++                return encodeIndex(i) ;
++        }
++        return encodeIndex(numSlot) ;
++    }
++
++    // Binary search
++    public int find(Record k) {
++        return find(k, 0, numSlot) ;
++    }
++
++    public Iterator<Record> iterator() {
++        return new RecordBufferIterator(this) ;
++    }
++
++    /** Iterator over a range from min (inclusive) to max(exclusive) */
++    public Iterator<Record> iterator(Record min, Record max) {
++        return new RecordBufferIterator(this, min, max) ;
++    }
++
++    public Record findGet(Record k) {
++        int x = find(k) ;
++        if ( x >= 0 )
++            return get(x) ;
++        return null ;
++    }
++
++    /** return true is removed anything */
++    public boolean removeByKey(Record k) {
++        int x = find(k) ;
++        if ( x < 0 )
++            return false ;
++        super.remove(x) ;
++        return true ;
++    }
++
++    /** Search for key in range fromIndex (inclusive) to toIndex (exclusive) */
++    public int find(Record rec, int fromIndex, int toIndex) {
++        int low = fromIndex ;
++        int high = toIndex - 1 ;
++
++        byte[] key = rec.getKey() ;
++        // http://en.wikipedia.org/wiki/Binary_search
++
++        while (low <= high) {
++            int mid = (low + high) >>> 1 ; // int divide by 2
++
++            int x = compare(mid, key) ;
++            // System.out.printf("Compare: %d(%s) %s ==> %d\n", mid,
++            // Record.str(get(mid)), Record.str(data), x) ;
++
++            if ( x < 0 )
++                low = mid + 1 ;
++            else if ( x > 0 )
++                high = mid - 1 ;
++            else
++                return mid ;
++        }
++        // On exit, when not finding, low is the least value
++        // above, including off the end of the array.
++        return encodeIndex(low) ;
++    }
++
++    // Record compareByKey except we avoid touching bytes by exiting as soon as
++    // possible.
++    // No record created as would be by using
++    // compareByKey(RecordBuffer.get(idx), record)
++    // Compare the slot at idx with value.
++    private int compare(int idx, byte[] value) {
++        idx = idx * slotLen ;
++
++        for ( int i = 0 ; i < value.length ; i++ ) {
++            byte b1 = bb.get(idx + i) ;
++            byte b2 = value[i] ;
++            if ( b1 == b2 )
++                continue ;
++            return (b1 & 0xFF) - (b2 & 0xFF) ;
++        }
++        return 0 ;
++    }
++
++    @Override
++    public String toString() {
++        StringBuilder str = new StringBuilder(40000) ;
++        str.append(format("Len=%d Max=%d: ", numSlot, bb.limit() / slotLen)) ;
++
++        // Print active slots as records.
++        for ( int i = 0 ; i < numSlot ; i++ ) {
++            if ( i != 0 )
++                str.append(" ") ;
++            Record r = _get(i) ;
++            str.append(r.toString()) ;
++        }
++
++        // // Print empty slots
++        // for ( int i = numSlot*slotLen ; i < maxSlot*slotLen ; i++ )
++        // {
++        // if ( i != 0 && i%slotLen == 0 )
++        // str.append(" ") ;
++        // byte b = bb.get(i) ;
++        // str.append(format("%02x", b)) ;
++        // }
++        String s = str.toString() ;
++        return s ;
++    }
++
++    private static void checkBounds(int idx, int len) {
++        if ( idx < 0 || idx >= len )
++            throw new IllegalArgumentException(format("Out of bounds: idx=%d, size=%d", idx, len)) ;
++    }
++
++    /**
++     * A duplicate which does not share anything with the original - for testing
++     */
++    public RecordBuffer duplicate() {
++        RecordBuffer n = new RecordBuffer(factory, maxSlot) ;
++        copy(0, n, 0, maxSlot) ;    // numSlot
++        n.numSlot = numSlot ;       // reset the allocated length
++        return n ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/RecordBufferIterator.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/RecordBufferIterator.java
index 0000000,0000000..bef7a7a
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/buffer/RecordBufferIterator.java
@@@ -1,0 -1,0 +1,97 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.buffer;
++
++import static org.apache.jena.atlas.lib.Alg.decodeIndex ;
++
++import java.util.Iterator;
++import java.util.NoSuchElementException;
++
++import org.seaborne.dboe.base.record.Record ;
++
++public class RecordBufferIterator implements Iterator<Record>
++{
++    private RecordBuffer rBuff ;
++    private int nextIdx ;
++    private Record slot = null ;
++    private final Record maxRec ;
++    private final Record minRec ;
++    
++    RecordBufferIterator(RecordBuffer rBuff)
++    { this(rBuff, null, null); }
++    
++    RecordBufferIterator(RecordBuffer rBuff, Record minRecord, Record maxRecord)
++    {
++        this.rBuff = rBuff ;
++        nextIdx = 0 ;
++        minRec = minRecord ;
++        if ( minRec != null )
++        {
++            nextIdx = rBuff.find(minRec) ;
++            if ( nextIdx < 0 )
++                nextIdx = decodeIndex(nextIdx) ;
++        }
++        
++        maxRec = maxRecord ; 
++    }
++
++    private void finish()
++    {
++        rBuff = null ;
++        nextIdx = -99 ;
++        slot = null ;
++    }
++    
++    @Override
++    public boolean hasNext()
++    {
++        if ( slot != null )
++            return true ;
++        if ( nextIdx < 0 )
++            return false ;
++        if ( nextIdx >= rBuff.size() )
++        {
++            finish() ;
++            return false ;
++        }
++        
++        slot = rBuff.get(nextIdx) ;
++        if ( maxRec != null && Record.keyGE(slot, maxRec) )
++        {
++            // Finished - now to large
++            finish() ;
++            return false ;
++        }
++        nextIdx ++ ;
++        return true ;
++    }
++
++    @Override
++    public Record next()
++    {
++        if ( ! hasNext() )
++            throw new NoSuchElementException("RecordBufferIterator") ;
++        Record r = slot ;
++        slot = null ;
++        return r ;
++    }
++
++    @Override
++    public void remove()
++    { throw new UnsupportedOperationException("RecordBufferIterator.remove") ; }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/AlreadyLocked.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/AlreadyLocked.java
index 0000000,0000000..dfe4125
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/AlreadyLocked.java
@@@ -1,0 -1,0 +1,29 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements.  See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership.  The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License.  You may obtain a copy of the License at
++ *
++ *     http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.dboe.base.file;
++
++import org.seaborne.dboe.DBOpEnvException;
++
++/** Exception throws when an attempt to take a {@link ProcessFileLock}
++ *  with a call of {@link ProcessFileLock#lockEx}
++ *  is made and the lock is already held.
++ */
++public class AlreadyLocked extends DBOpEnvException {
++    public AlreadyLocked(String msg) { super(msg); }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/BinaryDataFile.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/BinaryDataFile.java
index 0000000,0000000..3d9772c
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/BinaryDataFile.java
@@@ -1,0 -1,0 +1,99 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.file;
++
++import java.io.RandomAccessFile ;
++
++import org.apache.jena.atlas.lib.Closeable ;
++import org.apache.jena.atlas.lib.Sync ;
++
++/** An append-only, read-anywhere, binary file.
++ * A {@code BinaryDataFile} does not record the length and assumes the
++ * entries are self-defining.
++ * 
++ *  @see RandomAccessFile
++ */
++public interface BinaryDataFile extends Closeable, Sync {
++    // What about java.nio.channels.FileChannel?
++    // On OpenJDK,  RandomAccessFile and FileChannelImpl both dive into native code.
++    //  
++    // The choice seems to come down to ByteBuffers vs byte[]
++    // which in turn is a small/large data (scattered data)
++    // issue.  We are currently expecting small(er) I/O so byte[]
++    // and being like Thrift is better.
++
++    // byte[] vs ByteBuffer
++    
++    /** Open the file */
++    public void open() ;
++    
++    /** Is it open? */ 
++    public boolean isOpen() ;
++    
++    /** Read into a byte array, returning the number of bytes read. 
++     * Reads are at an absolute position and a read is atomic/thread-safe.
++     * 
++     * @param posn Location of the read operation.
++     * @param b byte array
++     * 
++     * @return The number of bytes read
++     */
++    public default int read(long posn, byte b[]) {
++        return read(posn, b, 0, b.length);
++    }
++
++    /** Read into a byte array, returning the number of bytes read.
++     * Reads are at an absolute position and a read is atomic/thread-safe.
++     * 
++     * @param posn Location of the read operation.
++     * @param b
++     * @param start of bytesarray to read into
++     * @param length Maximum number of bytes to read. 
++     * @return The number of bytes read
++     */
++ 
++    public int read(long posn, byte b[], int start, int length) ;
++
++    /** Write bytes - bytes are always written to the end of the file.
++     * Return the location where the write started.
++     */ 
++    public default long write(byte b[]) {
++        return write(b, 0, b.length) ;
++    }
++    
++    /** Write bytes - bytes are always written to the end of the file.
++     * Return the location where the write started.
++     */ 
++    public long write(byte b[], int start, int length) ;
++    
++    /** Return the length of the file (including any buffered writes) */
++    public long length() ;
++
++    /** Truncate the file */ 
++    public void truncate(long length) ; 
++
++    /** Return whether this is an empty file or not */ 
++    public default boolean isEmpty() { return length() == 0 ; } 
++
++    @Override
++    public void sync() ;
++    
++    @Override
++    public void close() ;
++}
++

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/BinaryDataFileMem.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/BinaryDataFileMem.java
index 0000000,0000000..17513a8
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/BinaryDataFileMem.java
@@@ -1,0 -1,0 +1,114 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.file;
++
++import org.apache.jena.atlas.RuntimeIOException ;
++import org.apache.jena.atlas.io.IO ;
++
++/** Implementation of {@link BinaryDataFile} in memory for testing
++ * and development use. Raw performance is not an objective.
++ * 
++ * <li>This implementation is thread-safe. 
++ */
++public class BinaryDataFileMem implements BinaryDataFile {
++
++    private boolean readMode ;
++    private SegmentedMemBuffer storage ;
++    
++    public BinaryDataFileMem() { 
++    }
++    
++    @Override
++    synchronized
++    public void open() {
++        if ( storage != null )
++            throw new RuntimeIOException("Already open") ;
++        storage = new SegmentedMemBuffer() ;
++        readMode = true ;
++    }
++
++    @Override
++    synchronized
++    public boolean isOpen() {
++        return storage != null ;
++    }
++    
++    @Override
++    synchronized
++    public int read(long posn, byte[] b, int start, int length) {
++        checkOpen() ;
++        switchToReadMode() ;
++        return storage.read(posn, b, start, length) ;
++    }
++
++    @Override
++    synchronized
++    public long write(byte[] b, int start, int length) {
++        checkOpen() ;
++        switchToWriteMode() ;
++        long x = storage.length() ;
++        storage.write(x, b, start, length) ;
++        return x ; 
++    }
++
++    @Override
++    synchronized
++    public void truncate(long length) {
++        if ( length < 0 )
++            IO.exception(String.format("truncate: bad length : %d", length)) ;
++        checkOpen() ;
++        switchToWriteMode() ;
++        storage.truncate(length); 
++    }
++
++    @Override
++    synchronized
++    public void sync() {
++        checkOpen() ;
++        storage.sync();
++    }
++
++    @Override
++    synchronized
++    public void close() {
++        if ( ! isOpen() )
++            return ;
++        storage.close();
++        storage = null ;
++    }
++
++    @Override
++    synchronized
++    public long length() {
++        return storage.length() ;
++    }
++    
++    private void switchToReadMode() {
++        readMode = true ;
++    }
++
++    private void switchToWriteMode() {
++        readMode = false ;
++    }
++
++    private void checkOpen() {
++        if ( ! isOpen() ) 
++            throw new RuntimeIOException("Not open") ;
++    }
++}
++