You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2017/09/28 11:08:45 UTC
[29/50] [abbrv] jena git commit: JENA-1396: Merge //github/afs/mantis
as subdirectory jena-db/
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/DoubleNode62.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/DoubleNode62.java
index 0000000,0000000..b65d5eb
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/DoubleNode62.java
@@@ -1,0 -1,0 +1,241 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.tdb2.store.value;
++
++import org.apache.jena.atlas.lib.BitsLong;
++
++/**
++ * Doubles packed into 62 bits.
++ * <p>
++ * Uses java's 64 bit long format (which is IEEE754 binary64) except that 2 bits are taken
++ * from the exponent. This keeps the precision but reduces the range.
++ * <p>
++ * <b>Java
++ * (<a href="https://en.wikipedia.org/wiki/Double-precision_floating-point_format">IEEE
++ * 754 binary64</a>)</b>
++ *
++ * <pre>
++ * bit 63 : sign bit
++ * bits 52-62 : exponent, 11 bits, the power of 2, bias -1023.
++ * bits 0-51 : mantissa (significand) 52 bits (the leading one is not stored).
++ *
++ * Exponents are 11 bits, with values -1022 to +1023 held as 1 to 2046 (11 bits, bias -1023)
++ * 0x000 is signed zero.
++ * 0x7FF is +/- infinity.
++ * </pre>
++ *
++ * for a maximum value of 1.797693e+308 = (2-2^-52)*2^1023 and smallest denormlized of
++ * (1-2^-52)*2^-1022 = 2.225...e-308.
++ * <p>
++ * <b>DoubleNode62</b>
++ * <p>
++ * In a 62 bit double:
++ *
++ * <pre>
++ * bit 63 : pointer bit.
++ * bit 62 : double type bit.
++ * bit 61 : sign bit
++ * bits 52-60 : exponent, 9 bits, the power of 2, bias -255
++ * bits 0-51 : mantissa (significand) 52 bits (the leading one is not stored).
++ *
++ * Exponents are 9 bits, with values -254 to 255, held as 1 to 512 (9 bits, bias -255)
++ * 0x000 is signed zero.
++ * 0x1FF is +/- infinity.
++ * </pre>
++ *
++ * for a maximum value of (2-2^-52)*2^255 = 1.157921e+77 and smallest denormlized of
++ * (1-2^-52)*2^-254 = 3.4544674e-77
++ * <p>
++ * 0 is not a legal encoding because the high bit is the pointer/value bit.
++ * <p>
++ * "No encoding" is 0xFF00_0000_0000_0000L which would otherwise be the smallest (most negative) denormalized value:
++ * -3.5336941295567687E72
++ * <p>All unencodeable numbers wil endup in the node table in full lexical form.
++ */
++public class DoubleNode62 {
++ /**
++ * An encoded value that is not possible.
++ * (it has high bits set).
++ */
++ public static long NO_ENCODING = 0xFF00_0000_0000_0000L;
++
++ /** Encode as a 62 bit long.
++ * Return {@link #NO_ENCODING} for "not possible".
++ * The top two bits are zero if packing was possible.
++ */
++ public static long pack(double v) {
++ long x = Double.doubleToRawLongBits(v);
++ long sign = BitsLong.unpack(x, 63, 64);
++ long exp11 = BitsLong.unpack(x, 52, 63);
++ long exp9 = encode11to9(exp11);
++ if ( exp9 == -1 )
++ return NO_ENCODING;
++ long significand = BitsLong.unpack(x, 0, 52);
++ // Do not set the value bit or double bit.
++ // This is done when outting into bytes.
++ long z = 0;
++ z = BitsLong.pack(z, sign, 61, 62);
++ z = BitsLong.pack(z, exp9, 52, 61);
++ z = BitsLong.pack(z, significand, 0, 52);
++ return z;
++ }
++
++ public static long insertType(long x) {
++ // bits 63 1 (pointer/value), bit 62 1(double)
++ return x | 0x3L<<62;
++ }
++
++ public static long removeType(long x) {
++ // bits 63 1 (pointer/value), bit 62 1(double)
++ return x & ~(0x3L<<62);
++ }
++
++ public static double unpack(long x) {
++ if ( x == NO_ENCODING )
++ throw new IllegalArgumentException("No encoding inline");
++ long sign = BitsLong.unpack(x, 61, 62);
++ long exp9 = BitsLong.unpack(x, 52, 61);
++ long significand = BitsLong.unpack(x, 0, 52);
++ long exp11 = decode9to11(exp9);
++ long z = 0;
++ z = BitsLong.pack(z, sign, 63, 64);
++ z = BitsLong.pack(z, exp11, 52, 63);
++ z = BitsLong.pack(z, significand, 0, 52);
++ double d = Double.longBitsToDouble(z);
++ return d;
++ }
++
++ // Exponent: returns -1 for out of inlien range.
++ private static long encode11to9(long exp11) {
++ if ( exp11 == 0 )
++ return 0L;
++ if ( exp11 == 0x7FF )
++ return 0x3FFL;
++
++ // Remove bias.
++ long expRebase = exp11 - 1023;
++ if ( expRebase < -254 || expRebase > 255 )
++ // Out of range.
++ return -1;
++ long exp9 = expRebase + 255;
++ return exp9;
++ }
++
++ // Exponent
++ private static long decode9to11(long exp9) {
++ if ( exp9 == 0 )
++ return 0L;
++ else if ( exp9 == 0x1FF )
++ return 0x7FFL;
++ long expRebase = exp9 - 255;
++ long exp11 = expRebase + 1023;
++ return exp11;
++
++ }
++
++ // ---- Constants without type bits (so bits 62 and 63 are zero).
++ /**
++ * 0x1ff0000000000000L
++ * @see Double#POSITIVE_INFINITY
++ */
++ public static final long POSITIVE_INFINITY_BITS = pack(Double.POSITIVE_INFINITY);
++
++ /**
++ * @see Double#POSITIVE_INFINITY
++ */
++ public static final double POSITIVE_INFINITY = unpack(POSITIVE_INFINITY_BITS);
++
++ /**
++ * 0x3ff0000000000000L
++ * @see Double#NEGATIVE_INFINITY
++ */
++ public static final long NEGATIVE_INFINITY_BITS = pack(Double.NEGATIVE_INFINITY);
++
++ /**
++ * @see Double#NEGATIVE_INFINITY
++ */
++ public static final double NEGATIVE_INFINITY = unpack(NEGATIVE_INFINITY_BITS);
++
++ /**
++ * 0x1ff8000000000000L
++ * @see Double#NaN
++ */
++ public static final long NaN_BITS = pack(Double.NaN);
++
++ /**
++ * @see Double#NaN
++ */
++ public static final double NaN = unpack(NaN_BITS);
++
++ /**
++ * 0x3fefffffffffffffL
++ * <br/>
++ * (2-2<sup>-52</sup>)·2<sup>255</sup>.
++ *
++ * @see Double#MAX_VALUE
++ */
++ public static final long MAX_VALUE_BITS = 0x3fefffffffffffffL;
++
++ /**
++ * 0x3fefffffffffffffL
++ * <br/>
++ * (2-2<sup>-52</sup>)·2<sup>255</sup>.
++ *
++ * @see Double#MAX_VALUE
++ */
++ public static final double MAX_VALUE = unpack(MAX_VALUE_BITS);
++
++ /**
++ * 0x0010000000000000L
++ * @see Double#MIN_NORMAL
++ */
++ public static final long MIN_NORMAL_BITS = 0x0010000000000000L;
++
++ /**
++ * @see Double#MIN_NORMAL
++ */
++ public static final double MIN_NORMAL = unpack(0x0010000000000000L);
++
++ /**
++ * 0x01L
++ * @see Double#MIN_VALUE
++ */
++ public static final long MIN_VALUE_BITS = 0x01L;
++
++ /**
++ * 0x01L
++ * @see Double#MIN_VALUE
++ */
++ public static final double MIN_VALUE = unpack(MIN_VALUE_BITS);
++
++ /**
++ * @see Double#MAX_EXPONENT
++ */
++ public static final int MAX_EXPONENT = 255;
++
++ /**
++ * @see Double#MIN_EXPONENT
++ */
++ public static final int MIN_EXPONENT = -254;
++
++ /**
++ * @see Double#SIZE
++ */
++ public static final int SIZE = 62;
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/FloatNode.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/FloatNode.java
index 0000000,0000000..4dc1c0a
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/FloatNode.java
@@@ -1,0 -1,0 +1,31 @@@
++/*
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ *
++ * See the NOTICE file distributed with this work for additional
++ * information regarding copyright ownership.
++ */
++
++package org.seaborne.tdb2.store.value;
++
++public class FloatNode {
++ // 32 bits of value; collapses NaNs to a single value.
++
++ public static long pack(float v) {
++ return Float.floatToIntBits(v);
++ }
++
++ public static float unpack(long v) {
++ return Float.intBitsToFloat((int)v);
++ }
++
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/IntegerNode.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/IntegerNode.java
index 0000000,0000000..9df7352
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/store/value/IntegerNode.java
@@@ -1,0 -1,0 +1,63 @@@
++/*
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ *
++ * See the NOTICE file distributed with this work for additional
++ * information regarding copyright ownership.
++ */
++
++package org.seaborne.tdb2.store.value;
++
++import org.apache.jena.atlas.lib.BitsLong;
++
++public class IntegerNode {
++ // 56 bits of value, including sign bit.
++ public static int LEN = 56;
++ public static int LBITS = Long.SIZE;
++ public static long MAX = (1L << (LEN - 1)) - 1;
++ public static long MIN = -(1L << (LEN - 1));
++
++ public static long pack(long v) {
++ if ( v >= MIN && v <= MAX ) {
++ v = BitsLong.clear(v, LEN, LBITS);
++ return v;
++ } else
++ return -1;
++ }
++
++ public static long unpack(long v) {
++ long val = BitsLong.clear(v, LEN, LBITS);
++ // Sign extends to 64 bits.
++ if ( BitsLong.isSet(val, LEN - 1) )
++ val = BitsLong.set(v, LEN, LBITS);
++ return val;
++ }
++
++ // Same - renamed.
++
++ public static long pack56(long v) {
++ if ( v >= MIN && v <= MAX ) {
++ v = BitsLong.clear(v, LEN, LBITS);
++ return v;
++ } else
++ return -1;
++ }
++
++ public static long unpack56(long v) {
++ long val = BitsLong.clear(v, LEN, LBITS);
++ // Sign extends to 64 bits.
++ if ( BitsLong.isSet(val, LEN - 1) )
++ val = BitsLong.set(v, LEN, LBITS);
++ return val;
++ }
++
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/ConnectionTracker.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/ConnectionTracker.java
index 0000000,0000000..f2950eb
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/ConnectionTracker.java
@@@ -1,0 -1,0 +1,23 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.tdb2.sys;
++
++abstract class ConnectionTracker<X> {
++
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/CopyDSG.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/CopyDSG.java
index 0000000,0000000..4e53ea3
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/CopyDSG.java
@@@ -1,0 -1,0 +1,53 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.util.Iterator;
++import java.util.List;
++
++import org.apache.jena.atlas.iterator.Iter;
++import org.apache.jena.graph.Graph;
++import org.apache.jena.graph.Node;
++import org.apache.jena.sparql.core.DatasetGraph;
++import org.apache.jena.sparql.core.Quad;
++import org.seaborne.dboe.jenax.Txn;
++
++/** Copy operations of any {@link DatasetGraph} */
++public class CopyDSG {
++
++ public static void copy(DatasetGraph dsgSrc, DatasetGraph dsgDst) {
++ Txn.executeRead(dsgSrc, ()->{
++ Txn.executeWrite(dsgDst, () -> {
++ Iterator<Quad> iter = dsgSrc.find();
++ iter.forEachRemaining(dsgDst::add);
++ copyPrefixes(dsgSrc, dsgDst);
++ });
++ });
++ }
++
++ public static void copyPrefixes(DatasetGraph dsgSrc, DatasetGraph dsgDst) {
++ List<Node> graphNames = Iter.toList(dsgSrc.listGraphNodes());
++ copyPrefixes(dsgSrc.getDefaultGraph(), dsgDst.getDefaultGraph());
++ graphNames.forEach((gn)->copyPrefixes(dsgSrc.getGraph(gn), dsgDst.getGraph(gn)));
++ }
++
++ public static void copyPrefixes(Graph srcGraph, Graph dstGraph) {
++ dstGraph.getPrefixMapping().setNsPrefixes(srcGraph.getPrefixMapping());
++ }
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/DatabaseConnection.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/DatabaseConnection.java
index 0000000,0000000..e9e6c27
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/DatabaseConnection.java
@@@ -1,0 -1,0 +1,180 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.io.IOException ;
++import java.nio.file.Path ;
++import java.nio.file.Paths ;
++import java.util.HashSet ;
++import java.util.Map ;
++import java.util.Set ;
++import java.util.concurrent.ConcurrentHashMap ;
++
++import org.apache.jena.atlas.io.IO ;
++import org.apache.jena.atlas.lib.FileOps ;
++import org.apache.jena.sparql.core.DatasetGraph ;
++import org.seaborne.dboe.base.file.Location ;
++import org.seaborne.dboe.base.file.ProcessFileLock ;
++import org.seaborne.dboe.sys.Names ;
++import org.seaborne.tdb2.TDBException ;
++import org.seaborne.tdb2.setup.StoreParams ;
++import org.seaborne.tdb2.store.DatasetGraphSwitchable ;
++
++// StoreConnection, DatabaseConnection < Connection<X>
++
++public class DatabaseConnection {
++ // ConnectionTracker<X>
++
++ private static Map<Location, DatabaseConnection> cache = new ConcurrentHashMap<>() ;
++
++ /** Get the {@code DatabaseConnection} to a location,
++ * creating the storage structures if it does not exist. */
++ public synchronized static DatabaseConnection connectCreate(Location location) {
++ return connectCreate(location, null) ;
++ }
++
++ /** Get the {@code DatabaseConnection} to a location,
++ * creating the storage structures if it does not exist.
++ * Use the provided {@link StoreParams} - any persistent setting
++ * already at the location take precedence.
++ */
++ public synchronized static DatabaseConnection connectCreate(Location location, StoreParams params) {
++ return make(location, params) ;
++ }
++
++ /**
++ * Return a {@code StoreConnection} for a particular location,
++ * creating it if it does not exist in storage.
++ */
++ private synchronized static DatabaseConnection make(Location location, StoreParams params) {
++ if ( location.isMemUnique() ) {
++ // Uncached, in-memory.
++ DatasetGraph dsg = DatabaseOps.create(location);
++ DatabaseConnection dbConn = new DatabaseConnection(dsg, location, null);
++ return dbConn;
++ }
++ // Cached by Location. Named in-memory or on-disk.
++ DatabaseConnection dbConn = cache.computeIfAbsent(location, (loc)->buildForCache(loc, params));
++ return dbConn ;
++ }
++
++ private static DatabaseConnection buildForCache(Location location, StoreParams params) {
++ if ( location.isMemUnique() ) {
++ throw new TDBException("Can't buildForCache a memory-unique location");
++ }
++ ProcessFileLock lock = null;
++ if (SystemTDB.DiskLocationMultiJvmUsagePrevention && ! location.isMem() ) {
++ lock = lockForLocation(location);
++ // Take the lock. This is atomic.
++ lock.lockEx();
++ }
++ DatasetGraph dsg = DatabaseOps.create(location);
++ return new DatabaseConnection(dsg, location, lock) ;
++ }
++
++// private static DatasetGraph buildMem(Location location, StoreParams params) {
++// return StoreConnection.connectCreate(location, params).getDatasetGraph();
++// }
++//
++// private static DatasetGraph buildDisk(Location location, StoreParams params) {
++// return DatabaseOps.create(location);
++// }
++
++ // DRY
++ /** Create or fetch a {@link ProcessFileLock} for a Location */
++ public static ProcessFileLock lockForLocation(Location location) {
++ FileOps.ensureDir(location.getDirectoryPath());
++ String lockFilename = location.getPath(Names.TDB_LOCK_FILE);
++ Path path = Paths.get(lockFilename);
++ try {
++ path.toFile().createNewFile();
++ } catch(IOException ex) { IO.exception(ex); return null; }
++ return ProcessFileLock.create(lockFilename);
++ }
++
++ /** Use via {@link TDBInternal#expel} wherever possible.
++ * <p>
++ * Stop managing a location. Use with great care (testing only).<br/>
++ * Does not expel from {@link StoreConnection}.
++ */
++ public static synchronized void internalExpel(Location location, boolean force) {
++ // ** Check it is a container location
++ DatabaseConnection dbConn = cache.get(location) ;
++ if ( dbConn == null )
++ return ;
++ dbConn.isValid = false;
++ //dbConn.datasetGraph = null;
++ //dbConn.datasetGraphSwitchable = null;
++ cache.remove(location) ;
++ // Release the lock after the cache is emptied.
++ if (SystemTDB.DiskLocationMultiJvmUsagePrevention && ! location.isMem() ) {
++ if ( ! dbConn.lock.isLockedHere() )
++ SystemTDB.errlog.warn("Location " + location.getDirectoryPath() + " was not locked by this process.");
++ dbConn.lock.unlock();
++ ProcessFileLock.release(dbConn.lock);
++ }
++ }
++
++ /**
++ * Stop managing all locations.
++ * Use with extreme care.
++ * This is intended to support internal testing.
++ */
++ public static synchronized void internalReset() {
++ // Copy to avoid potential CME.
++ Set<Location> x = new HashSet<>(cache.keySet()) ;
++ for (Location loc : x)
++ internalExpel(loc, true) ;
++ if ( ! cache.isEmpty() )
++ System.err.println("DatabaseConnection: Cache not empty!");
++ cache.clear() ;
++ }
++
++ // One of the other.
++ private final DatasetGraphSwitchable datasetGraphSwitchable;
++ private final DatasetGraph datasetGraph;
++ // This is the location of the TDB2 container directory.
++ private final Location location ;
++ private final ProcessFileLock lock ;
++ private boolean isValid = true ;
++
++ private DatabaseConnection(DatasetGraph dsg, Location location, ProcessFileLock fileLock)
++ {
++ this.datasetGraph = dsg;
++ this.datasetGraphSwitchable = ( dsg instanceof DatasetGraphSwitchable ) ? (DatasetGraphSwitchable )dsg : null;
++ this.location = location ;
++ this.lock = fileLock;
++ }
++
++ public DatasetGraph getDatasetGraph() {
++ return datasetGraph;
++ }
++
++ public DatasetGraphSwitchable getDatasetGraphSwitchable() {
++ return datasetGraphSwitchable;
++ }
++
++ public Location getLocation() {
++ return location ;
++ }
++
++ public ProcessFileLock getLock() {
++ return lock ;
++ }
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/DatabaseOps.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/DatabaseOps.java
index 0000000,0000000..bb34ed3
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/DatabaseOps.java
@@@ -1,0 -1,0 +1,264 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.io.BufferedOutputStream ;
++import java.io.IOException ;
++import java.io.OutputStream ;
++import java.nio.file.* ;
++import java.util.List;
++import java.util.zip.GZIPOutputStream ;
++
++import org.apache.jena.atlas.RuntimeIOException ;
++import org.apache.jena.atlas.lib.DateTimeUtils ;
++import org.apache.jena.atlas.lib.Pair ;
++import org.apache.jena.atlas.logging.Log;
++import org.apache.jena.riot.Lang ;
++import org.apache.jena.riot.RDFDataMgr ;
++import org.apache.jena.sparql.core.DatasetGraph;
++import org.seaborne.dboe.base.file.Location;
++import org.seaborne.dboe.jenax.Txn ;
++import org.seaborne.dboe.transaction.txn.TransactionCoordinator;
++import org.seaborne.dboe.transaction.txn.TransactionalSystem;
++import org.seaborne.tdb2.TDBException;
++import org.seaborne.tdb2.setup.StoreParams;
++import org.seaborne.tdb2.store.DatasetGraphSwitchable ;
++import org.seaborne.tdb2.store.DatasetGraphTDB;
++import org.slf4j.Logger ;
++import org.slf4j.LoggerFactory ;
++
++/** Operations on and about TDB2 databases.
++ * <p>
++ * TDB2 uses a hierarchical structure to manage on disk.
++ * <p>
++ * If in-memory (non-scalable, not-performant, perfect simulation for functionality using a RAM disk, for testing mainly),
++ * then the switchable layer is just a covenience. DatasetGrapOps such as {@link #compact}
++ * and {@link #backup} do not apply.
++ * <p>
++ * Directory and files on disk:
++ * <ul>
++ * <li> {@code Data-NNNN/} -- databases by version. Compacting creates a new directory, leaving
++ * <li> store params, static (disk layout related - can not chnage once created) and dynamic (settings related to in-memory).
++ * <li> {@code Backups/} -- backups.
++ * <li> External indexes like {@code TextIndex/} -- the text index only applies to the current, latest database.
++ * </ul>
++ */
++public class DatabaseOps {
++ private static Logger LOG = LoggerFactory.getLogger(DatabaseOps.class);
++ private static final String dbPrefix = "Data";
++ private static final String SEP = "-";
++ private static final String startCount = "0001";
++
++ private static final String BACKUPS_DIR = "Backups";
++ // Basename of the backup file. "backup_{DateTime}.nq.gz
++ private static final String BACKUPS_FN = "backup";
++
++ /** Create a fresh database - called by {@code DatabaseMgr}.
++ * It is important to go via {@code DatabaseConnection} to avoid
++ * duplicate {@code DatasetGraphSwitchable}s for the same location.
++ */
++ /*package*/ static DatasetGraph create(Location location) {
++ // Hide implementation class.
++ return createSwitchable(location);
++ }
++
++ private static DatasetGraphSwitchable createSwitchable(Location location) {
++ if ( location.isMem() ) {
++ DatasetGraph dsg = StoreConnection.connectCreate(location).getDatasetGraph();
++ return new DatasetGraphSwitchable(null, location, dsg);
++ }
++ // Exists?
++ if ( ! location.exists() )
++ throw new TDBException("No such location: "+location);
++ Path path = IOX.asPath(location);
++ // Scan for DBs
++ Path db = findLocation(path, dbPrefix);
++ // XXX [StoreParams]
++ StoreParams params;
++ if ( db == null ) {
++ db = path.resolve(dbPrefix+SEP+startCount);
++ IOX.createDirectory(db);
++ }
++ Location loc2 = IOX.asLocation(db);
++ DatasetGraphTDB dsg = StoreConnection.connectCreate(loc2).getDatasetGraphTDB();
++ DatasetGraphSwitchable appDSG = new DatasetGraphSwitchable(path, location, dsg);
++ return appDSG;
++ }
++
++ public static String backup(DatasetGraphSwitchable container) {
++ checkSupportsAdmin(container);
++ Path dbPath = container.getContainerPath();
++ Path backupDir = dbPath.resolve(BACKUPS_DIR);
++ if ( ! Files.exists(backupDir) )
++ IOX.createDirectory(backupDir);
++
++ DatasetGraph dsg = container;
++
++// // Per backup source lock.
++// synchronized(activeBackups) {
++// // Atomically check-and-set
++// if ( activeBackups.contains(dsg) )
++// Log.warn(Fuseki.serverLog, "Backup already in progress") ;
++// activeBackups.add(dsg) ;
++// }
++
++ Pair<OutputStream, Path> x = openUniqueFileForWriting(backupDir, BACKUPS_FN, "nq.gz");
++ try (OutputStream out2 = x.getLeft();
++ OutputStream out1 = new GZIPOutputStream(out2, 8 * 1024) ;
++ OutputStream out = new BufferedOutputStream(out1)) {
++ Txn.executeRead(dsg, ()->RDFDataMgr.write(out, dsg, Lang.NQUADS));
++ } catch (IOException e) {
++ throw IOX.exception(e) ;
++ }
++ return x.getRight().toString();
++ }
++
++ private static void checkSupportsAdmin(DatasetGraphSwitchable container) {
++ if ( ! container.hasContainerPath() )
++ throw new TDBException("Dataset does not support admin operations");
++ }
++
++ //private static void logWarn(String msg) {}
++
++
++ // --> IOX
++ private static Pair<OutputStream, Path> openUniqueFileForWriting(Path dirPath, String basename, String ext) {
++ if ( ! Files.isDirectory(dirPath) )
++ throw new IllegalArgumentException("Not a directory: "+dirPath);
++ if ( basename.contains("/") || basename.contains("\\") )
++ throw new IllegalArgumentException("Basename must not contain a file path separator (\"/\" or \"\\\")");
++
++ String timestamp = DateTimeUtils.nowAsString("yyyy-MM-dd_HH:mm:ss") ;
++ String filename = basename + "_" + timestamp ;
++ Path p = dirPath.resolve(filename+"."+ext);
++ int x = 0 ;
++ for(;;) {
++ try {
++ OutputStream out = Files.newOutputStream(p, StandardOpenOption.CREATE_NEW);
++ return Pair.create(out, p);
++ } catch (AccessDeniedException ex) {
++ throw IOX.exception("Access denied", ex);
++ } catch (FileAlreadyExistsException ex) {
++ // Drop through and try again.
++ } catch (IOException ex) {
++ throw IOX.exception(ex) ;
++ }
++ // Try again.
++ x++;
++ if ( x >= 5 )
++ throw new RuntimeIOException("Can't create the unique name: number of attempts exceeded");
++ p = dirPath.resolve(filename+"_"+x+"."+ext);
++ }
++ }
++
++
++ // JVM-wide :-(
++ private static Object compactionLock = new Object();
++
++ public static void compact(DatasetGraphSwitchable container) {
++ checkSupportsAdmin(container);
++ synchronized(compactionLock) {
++ Path base = container.getContainerPath();
++ Path db1 = findLocation(base, dbPrefix);
++ Location loc1 = IOX.asLocation(db1);
++
++ // -- Checks
++ Location loc1a = ((DatasetGraphTDB)container.get()).getLocation();
++ if ( loc1a.isMem() ) {}
++ if ( ! loc1a.exists() )
++ throw new TDBException("No such location: "+loc1a);
++
++ // Is this the same database location?
++ if ( ! loc1.equals(loc1a) )
++ throw new TDBException("Inconsistent (not latested?) : "+loc1a+" : "+loc1);
++ // -- Checks
++
++ // Version
++ int v = IOX.extractIndex(db1.getFileName().toString(), dbPrefix, SEP);
++ String next = FilenameUtils.filename(dbPrefix, SEP, v+1);
++
++ Path db2 = db1.getParent().resolve(next);
++ IOX.createDirectory(db2);
++ Location loc2 = IOX.asLocation(db2);
++ LOG.debug(String.format("Compact %s -> %s\n", db1.getFileName(), db2.getFileName()));
++
++ compact(container, loc1, loc2);
++ }
++ }
++
++ // XXX Later - switch in a recording dataset, not block writers, and reply after
++ // switch over before releasing the new dataset to the container.
++ // Maybe copy indexes and switch the DSG over (drop switchable).
++
++ /** Copy the latest version from one location to another. */
++ private static void compact(DatasetGraphSwitchable container, Location loc1, Location loc2) {
++ if ( loc1.isMem() || loc2.isMem() )
++ throw new TDBException("Compact involves a memory location: "+loc1+" : "+loc2);
++ StoreConnection srcConn = StoreConnection.connectExisting(loc1);
++ if ( srcConn == null ) {
++ throw new TDBException("No database at location : "+loc1);
++ }
++
++ DatasetGraphTDB dsgCurrent = (DatasetGraphTDB)container.get();
++ if ( ! dsgCurrent.getLocation().equals(loc1) )
++ throw new TDBException("Inconsistent locations for base : "+dsgCurrent.getLocation()+" , "+dsgCurrent.getLocation());
++
++ DatasetGraphTDB dsgBase = srcConn.getDatasetGraphTDB();
++ if ( dsgBase != dsgCurrent )
++ throw new TDBException("Inconsistent datasets : "+dsgCurrent.getLocation()+" , "+dsgBase.getLocation());
++
++ TransactionalSystem txnSystem = dsgBase.getTxnSystem();
++ TransactionCoordinator txnMgr = dsgBase.getTxnSystem().getTxnMgr();
++
++ // Stop update. On exit there are no writers and none will start until switched over.
++ txnMgr.tryBlockWriters();
++ // txnMgr.begin(WRITE, false) will now bounce.
++
++ // Copy the latest generation.
++ DatasetGraphTDB dsgCompact = StoreConnection.connectCreate(loc2).getDatasetGraphTDB();
++ CopyDSG.copy(dsgBase, dsgCompact);
++
++ TransactionCoordinator txnMgr2 = dsgCompact.getTxnSystem().getTxnMgr();
++ txnMgr2.startExclusiveMode();
++
++ txnMgr.startExclusiveMode();
++
++ // No transactions on either database.
++ // Switch.
++ if ( ! container.change(dsgCurrent, dsgCompact) ) {
++ Log.warn(DatabaseOps.class, "Inconistent: old datasetgraph not as expected");
++ container.set(dsgCompact);
++ }
++ txnMgr2.finishExclusiveMode();
++ // New database running.
++
++ // Clean-up.
++ // txnMgr.finishExclusiveMode();
++ // Don't call : txnMgr.startWriters();
++ StoreConnection.release(dsgBase.getLocation());
++ }
++
++ private static Path findLocation(Path directory, String namebase) {
++ if ( ! Files.exists(directory) )
++ return null;
++ // In-order, low to high.
++ List<Path> maybe = IOX.scanForDirByPattern(directory, namebase, SEP);
++ return Util.getLastOrNull(maybe);
++ }
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/EnvTDB.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/EnvTDB.java
index 0000000,0000000..449853e
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/EnvTDB.java
@@@ -1,0 -1,0 +1,61 @@@
++/*
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ *
++ * See the NOTICE file distributed with this work for additional
++ * information regarding copyright ownership.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.util.Properties ;
++import java.util.Set ;
++
++import org.apache.jena.sparql.util.Context ;
++import org.apache.jena.sparql.util.Symbol ;
++import org.seaborne.tdb2.TDB2 ;
++
++public class EnvTDB
++{
++ public static void processGlobalSystemProperties()
++ {
++ Context context = processProperties(System.getProperties()) ;
++ TDB2.getContext().putAll(context) ;
++ }
++
++ static final String prefix = SystemTDB.tdbSymbolPrefix+":" ;
++ public static Context processProperties(Properties properties)
++ {
++ Context context = new Context() ;
++ Set<Object> keys = properties.keySet() ;
++ for ( Object key : keys )
++ {
++ if ( key instanceof String )
++ {
++ String keyStr = (String)key ;
++ if ( keyStr.startsWith(prefix) )
++ keyStr = SystemTDB.symbolNamespace+keyStr.substring(prefix.length()) ;
++
++
++ if ( ! keyStr.startsWith(SystemTDB.symbolNamespace) )
++ continue ;
++
++ Object value = properties.get(key) ;
++
++ Symbol symbol = Symbol.create(keyStr) ;
++
++ context.set(symbol, value) ;
++ }
++ }
++ return context ;
++ }
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/FilenameUtils.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/FilenameUtils.java
index 0000000,0000000..623ac99
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/FilenameUtils.java
@@@ -1,0 -1,0 +1,102 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.io.IOException;
++import java.nio.file.DirectoryStream;
++import java.nio.file.Files;
++import java.nio.file.Path;
++import java.util.ArrayList;
++import java.util.Comparator;
++import java.util.List;
++import java.util.regex.Pattern;
++
++import org.apache.jena.atlas.logging.FmtLog;
++import org.seaborne.dboe.DBOpEnvException;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
++
++public class FilenameUtils {
++
++ static Logger LOG = LoggerFactory.getLogger(FilenameUtils.class);
++
++ /** Find the files in this directory that have namebase as a prefix and
++ * are then numbered.
++ * <p>
++ * Returns a sorted list from, low to high index.
++ */
++ public static List<Path> scanForDirByPattern(Path directory, String namebase, String nameSep) {
++ Pattern pattern = Pattern.compile(Pattern.quote(namebase)+
++ Pattern.quote(nameSep)+
++ "[\\d]+");
++ List<Path> paths = new ArrayList<>();
++ try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory, namebase + nameSep + "*")) {
++ for ( Path entry : stream ) {
++ if ( !pattern.matcher(entry.getFileName().toString()).matches() ) {
++ throw new DBOpEnvException("Invalid filename for matching: "+entry.getFileName());
++ // Alternative: Skip bad trailing parts but more likely there is a naming problem.
++ // LOG.warn("Invalid filename for matching: {} skipped", entry.getFileName());
++ // continue;
++ }
++ // Follows symbolic links.
++ if ( !Files.isDirectory(entry) )
++ throw new DBOpEnvException("Not a directory: "+entry);
++ paths.add(entry);
++ }
++ }
++ catch (IOException ex) {
++ FmtLog.warn(LOG, "Can't inspect directory: (%s, %s)", directory, namebase);
++ throw new DBOpEnvException(ex);
++ }
++ Comparator<Path> comp = (f1, f2) -> {
++ int num1 = extractIndex(f1.getFileName().toString(), namebase, nameSep);
++ int num2 = extractIndex(f2.getFileName().toString(), namebase, nameSep);
++ return Integer.compare(num1, num2);
++ };
++ paths.sort(comp);
++ //indexes.sort(Long::compareTo);
++ return paths;
++ }
++
++ /**
++ * Extract the inde from a version-ed filename. (Base-NNNN format).
++ * @param name
++ * @param namebase
++ * @param nameSep
++ */
++ public static int extractIndex(String name, String namebase, String nameSep) {
++ int i = namebase.length()+nameSep.length();
++ String numStr = name.substring(i);
++ int num = Integer.parseInt(numStr);
++ return num;
++ }
++
++ /** Construct a filename */
++ public static String filename(String prefix, String sep, int N) {
++ return String.format("%s%s%04d", prefix, sep, N);
++ }
++
++ /** Construct a filename */
++ public static String filename(String prefix, String sep, String index) {
++ return String.format("%s%s%s", prefix, sep, index);
++ }
++
++
++
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/IOX.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/IOX.java
index 0000000,0000000..ed78f88
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/IOX.java
@@@ -1,0 -1,0 +1,292 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.io.BufferedOutputStream;
++import java.io.File;
++import java.io.IOException;
++import java.io.OutputStream;
++import java.nio.file.*;
++import java.nio.file.attribute.FileAttribute;
++import java.util.ArrayList;
++import java.util.Comparator;
++import java.util.List;
++import java.util.function.Function ;
++import java.util.regex.Pattern;
++
++import org.apache.jena.atlas.RuntimeIOException;
++import org.apache.jena.atlas.logging.FmtLog;
++import org.seaborne.dboe.DBOpEnvException;
++import org.seaborne.dboe.base.file.Location;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
++
++/**
++ * PathX
++ */
++public class IOX {
++
++ public static Path currentDirectory = Paths.get(".");
++
++ /** A Consumer that can throw {@link IOException}. */
++ public interface IOConsumer<X> {
++ void actionEx(X arg) throws IOException;
++ }
++
++ /** Convert an {@link IOException} into a {@link RuntimeIOException}.
++ * <p>
++ * Idiom:
++ * <pre>
++ * catch(IOException ex) { throw new exception(ex); }
++ * </pre>
++ * @param ioException
++ * @return RuntimeIOException
++ */
++ public static RuntimeIOException exception(IOException ioException) {
++ return new RuntimeIOException(ioException);
++ }
++
++ /** Convert an {@link IOException} into a {@link RuntimeIOException}.
++ * <p>
++ * Idiom:
++ * <pre>
++ * catch(IOException ex) { throw new exception("Oh dear", ex); }
++ * </pre>
++ * @param message
++ * @param ioException
++ * @return RuntimeIOException
++ */
++ public static RuntimeIOException exception(String message, IOException ioException) {
++ return new RuntimeIOException(message, ioException);
++ }
++
++ /** Write a file safely - the change happens (the function returns true) or
++ * somthing went wrong (the function throws a runtime exception) and the file is not changed.
++ * Note that the tempfile must be in the same direct as the actual file so an OS-atomic rename can be done.
++ */
++ public static boolean safeWrite(Path file, IOConsumer<OutputStream> writerAction) {
++ Path tmp = createTempFile(file.getParent(), file.getFileName().toString(), ".tmp");
++ return safeWrite(file, tmp, writerAction);
++ }
++
++ /** Write a file safely - the change happens (the function returns true) or
++ * somthing went wrong (the function throws a runtime exception) and the file is not changed.
++ * Note that the tempfile must be in the same direct as the actual file so an OS-atomic rename can be done.
++ */
++ public static boolean safeWrite(Path file, Path tmpFile, IOConsumer<OutputStream> writerAction) {
++ try {
++ try(OutputStream out = new BufferedOutputStream(Files.newOutputStream(tmpFile)) ) {
++ writerAction.actionEx(out);
++ }
++ move(tmpFile, file);
++ return true;
++ } catch(IOException ex) { throw IOX.exception(ex); }
++ }
++
++ /** Delete a file. */
++ public static void delete(Path path) {
++ try { Files.delete(path); }
++ catch (IOException ex) {
++ FmtLog.error(IOX.class, ex, "IOException deleting %s", path);
++ throw IOX.exception(ex);
++ }
++ }
++
++ /** Atomically move a file. */
++ public static void move(Path src, Path dst) {
++ try { Files.move(src, dst, StandardCopyOption.ATOMIC_MOVE) ; }
++ catch (IOException ex) {
++ FmtLog.error(IOX.class, ex, "IOException moving %s to %s", src, dst);
++ throw IOX.exception(ex);
++ }
++ }
++
++ /** Copy a file, not atomic. *
++ * Can copy to a directory or over an existing file.
++ * @param srcFilename
++ * @param dstFilename
++ */
++ public static void copy(String srcFilename, String dstFilename) {
++ Path src = Paths.get(srcFilename);
++ if ( ! Files.exists(src) )
++ throw new RuntimeIOException("No such file: "+srcFilename);
++
++ Path dst = Paths.get(dstFilename);
++ if ( Files.isDirectory(dst) )
++ dst = dst.resolve(src.getFileName());
++
++ try { Files.copy(src, dst); }
++ catch (IOException ex) {
++ FmtLog.error(IOX.class, ex, "IOException copying %s to %s", srcFilename, dstFilename);
++ throw IOX.exception(ex);
++ }
++ }
++
++ /** Create a directory - thgrow a runtime exception if theer are any problems.
++ * This function wraps {@code Files.createDirectory}.
++ */
++ public static void createDirectory(Path dir) {
++ try { Files.createDirectory(dir); }
++ catch (IOException ex) { throw IOX.exception(ex); }
++ }
++
++ /** Convert a {@link Path} to a {@link Location}. */
++ public static Location asLocation(Path path) {
++ if ( ! Files.isDirectory(path) )
++ throw new RuntimeIOException("Path is not naming a directory: "+path);
++ return Location.create(path.toString());
++ }
++
++// /** Convert a {@link org.apache.jena.tdb.base.file.Location} to a {@link Path}. */
++// public static Path asPath(org.apache.jena.tdb.base.file.Location location) {
++// if ( location.isMem() )
++// throw new RuntimeIOException("Location is a memory location: "+location);
++// return Paths.get(location.getDirectoryPath());
++// }
++
++ /** Convert a {@link Location} to a {@link Path}. */
++ public static Path asPath(org.seaborne.dboe.base.file.Location location) {
++ if ( location.isMem() )
++ throw new RuntimeIOException("Location is a memory location: "+location);
++ return Paths.get(location.getDirectoryPath());
++ }
++
++ /** Convert a {@link Location} to a {@link File}. */
++ public static File asFile(Location loc) {
++ return new File(loc.getDirectoryPath());
++ }
++
++ /** Read the whole of a file */
++ public static byte[] readAll(Path pathname) {
++ try {
++ return Files.readAllBytes(pathname);
++ } catch (IOException ex) { throw IOX.exception(ex); }
++ }
++
++ /** Write the whole of a file */
++ public static void writeAll(Path pathname, byte[] value) {
++ try {
++ Files.write(pathname, value, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
++ } catch (IOException ex) { throw IOX.exception(ex); }
++ }
++
++// /**
++// * Ensure a directory exists, creating the directory and any missing
++// * directories on the path to it.
++// */
++// public static void ensureDirectory(Path dir, FileAttribute<? >... attrs) {
++// if ( Files.exists(dir) ) {
++// if ( ! Files.isDirectory(dir) )
++// throw new DeltaFileException("Exists but not a directory: "+dir);
++// }
++// try { Files.createDirectories(dir, attrs); }
++// catch (IOException ex) { new DeltaFileException("Failed to create directory: "+dir, ex);}
++// }
++//
++// /**
++// * Ensure a file exists - create an empty one if not. This operation does
++// * not create a directory path to the file.
++// */
++// public static void ensureFile(Path filePath, FileAttribute<? >... attrs) {
++// if ( Files.exists(filePath) ) {
++// if ( ! Files.isRegularFile(filePath) )
++// throw new DeltaFileException("Exists but not a regular file: "+filePath);
++// }
++// try { Files.createFile(filePath); }
++// catch (IOException ex) { new DeltaFileException("Failed to create file: "+filePath, ex);}
++// }
++
++ /**
++ * Return a temporary filename path.
++ * <p>
++ * This operation is thread-safe.
++ */
++ public static Path createTempFile(Path dir, String prefix, String suffix, FileAttribute<? >... attrs) {
++ try {
++ return Files.createTempFile(dir, prefix, suffix, attrs);
++ } catch (IOException ex) { throw IOX.exception(ex); }
++ }
++
++ /** Generate a unique place related to path;
++ * Optionally, provide a mapping of old name to new nae base.
++ * This method always adds "-1", "-2" etc.
++ */
++ public static Path uniqueDerivedPath(Path path, Function<String, String> basenameMapping) {
++ String base = path.getFileName().toString();
++ if ( basenameMapping != null )
++ base = basenameMapping.apply(base);
++ // Some large limit "just in case"
++ for(int x = 0 ; x < 10_000 ; x++ ) {
++ String destname = base+"-"+x;
++ Path destpath = path.resolveSibling(destname);
++ if ( ! Files.exists(destpath) )
++ return destpath;
++ }
++ return null ;
++ }
++
++ private static Logger LOG = LoggerFactory.getLogger(Util.class);
++
++ /** Find the files in this directory that have namebase as a prefix and
++ * are then numbered.
++ * <p>
++ * Returns a sorted list from, low to high index.
++ */
++ public static List<Path> scanForDirByPattern(Path directory, String namebase, String nameSep) {
++ Pattern pattern = Pattern.compile(Pattern.quote(namebase)+
++ Pattern.quote(nameSep)+
++ "[\\d]+");
++ List<Path> paths = new ArrayList<>();
++ try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory, namebase + "*")) {
++ for ( Path entry : stream ) {
++ if ( !pattern.matcher(entry.getFileName().toString()).matches() ) {
++ throw new DBOpEnvException("Invalid filename for matching: "+entry.getFileName());
++ // Alternative: Skip bad trailing parts but more likely there is a naming problem.
++ // LOG.warn("Invalid filename for matching: {} skipped", entry.getFileName());
++ // continue;
++ }
++ // Follows symbolic links.
++ if ( !Files.isDirectory(entry) )
++ throw new DBOpEnvException("Not a directory: "+entry);
++ paths.add(entry);
++ }
++ }
++ catch (IOException ex) {
++ FmtLog.warn(LOG, "Can't inspect directory: (%s, %s)", directory, namebase);
++ throw new DBOpEnvException(ex);
++ }
++ Comparator<Path> comp = (f1, f2) -> {
++ int num1 = extractIndex(f1.getFileName().toString(), namebase, nameSep);
++ int num2 = extractIndex(f2.getFileName().toString(), namebase, nameSep);
++ return Integer.compare(num1, num2);
++ };
++ paths.sort(comp);
++ //indexes.sort(Long::compareTo);
++ return paths;
++ }
++
++ /** Given a filename in "base-NNNN" format, return the value of NNNN */
++ public static int extractIndex(String name, String namebase, String nameSep) {
++ int i = namebase.length()+nameSep.length();
++ String numStr = name.substring(i);
++ int num = Integer.parseInt(numStr);
++ return num;
++ }
++
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/InitTDB2.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/InitTDB2.java
index 0000000,0000000..5d82fe3
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/InitTDB2.java
@@@ -1,0 -1,0 +1,40 @@@
++/*
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ *
++ * See the NOTICE file distributed with this work for additional
++ * information regarding copyright ownership.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import org.apache.jena.system.JenaSubsystemLifecycle ;
++import org.seaborne.tdb2.TDB2 ;
++
++public class InitTDB2 implements JenaSubsystemLifecycle {
++
++ @Override
++ public void start() {
++ TDB2.init() ;
++ }
++
++ @Override
++ public void stop() {
++ // This is savage and does not take account of in-flight transactions.
++ TDB2.closedown() ;
++ }
++
++ @Override
++ public int level() {
++ return 42 ;
++ }
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/LockMRSWLite.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/LockMRSWLite.java
index 0000000,0000000..cf381ef
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/LockMRSWLite.java
@@@ -1,0 -1,0 +1,69 @@@
++/*
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ *
++ * See the NOTICE file distributed with this work for additional
++ * information regarding copyright ownership.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.util.concurrent.locks.ReadWriteLock ;
++import java.util.concurrent.locks.ReentrantReadWriteLock ;
++
++import org.apache.jena.shared.JenaException ;
++import org.apache.jena.shared.Lock ;
++
++/** Light weight (?) MRSW lock implementation that assumes
++ * correct use of enterCriticalSection/leaveCriticalSection.
++ * That is, there is no real checking.
++ */
++public class LockMRSWLite implements Lock
++{
++ public LockMRSWLite() {}
++
++ private ReadWriteLock mrswLock = new ReentrantReadWriteLock() ;
++ // >0 for read lock, -1 for write lock.
++ private int count = 0 ;
++
++ @Override
++ public synchronized void enterCriticalSection(boolean readLockRequested)
++ {
++ // Once we have the lock, we can record the lock state
++ // because we know whether the actiev thread (us) is a read or write
++ // operation, then a valid leaveCriticalSection can only be read or
++ // write.
++ if ( readLockRequested ) {
++ mrswLock.readLock().lock() ;
++ count++ ;
++ } else {
++ mrswLock.writeLock().lock() ;
++ count = -1 ;
++ }
++ }
++
++ @Override
++ public synchronized void leaveCriticalSection()
++ {
++ if ( count == 0 )
++ throw new JenaException("Bad lock release - don't appear to be in a critical section") ;
++
++ if ( count < 0 ) {
++ mrswLock.writeLock().unlock() ;
++ count = 0 ;
++ return ;
++ } else {
++ mrswLock.readLock().unlock() ;
++ count-- ;
++ }
++ }
++}
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/StoreConnection.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/StoreConnection.java
index 0000000,0000000..21d5377
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/StoreConnection.java
@@@ -1,0 -1,0 +1,194 @@@
++/*
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ *
++ * See the NOTICE file distributed with this work for additional
++ * information regarding copyright ownership.
++ */
++
++package org.seaborne.tdb2.sys ;
++
++import java.io.IOException;
++import java.nio.file.Path;
++import java.nio.file.Paths;
++import java.util.HashSet ;
++import java.util.Map ;
++import java.util.Set ;
++import java.util.concurrent.ConcurrentHashMap ;
++
++import org.apache.jena.atlas.io.IO;
++import org.apache.jena.atlas.lib.FileOps;
++import org.apache.jena.sparql.core.DatasetGraph ;
++import org.seaborne.dboe.base.file.ChannelManager ;
++import org.seaborne.dboe.base.file.Location ;
++import org.seaborne.dboe.base.file.ProcessFileLock;
++import org.seaborne.dboe.sys.Names;
++import org.seaborne.dboe.transaction.txn.TransactionCoordinator ;
++import org.seaborne.dboe.transaction.txn.TransactionException ;
++import org.seaborne.tdb2.setup.StoreParams ;
++import org.seaborne.tdb2.setup.TDBBuilder ;
++import org.seaborne.tdb2.store.DatasetGraphTDB ;
++
++/** A StoreConnection is the reference to the underlying storage.
++ * There is JVM-wide cache of backing datasets.
++ */
++public class StoreConnection
++{
++ private static Map<Location, StoreConnection> cache = new ConcurrentHashMap<>() ;
++
++ /** Get the {@code StoreConnection} to a location,
++ * creating the storage structures if it does not exist. */
++ public synchronized static StoreConnection connectCreate(Location location) {
++ return connectCreate(location, null) ;
++ }
++
++ /** Get the {@code StoreConnection} to a location,
++ * creating the storage structures if it does not exist.
++ * Use the provided {@link StoreParams} - any persistent setting
++ * already at the location take precedence.
++ */
++ public synchronized static StoreConnection connectCreate(Location location, StoreParams params) {
++ return make(location, params) ;
++ }
++
++ /** Get the {@code StoreConnection} for a location, but do not create it.
++ * Returns null for "not setup".
++ */
++ public synchronized static StoreConnection connectExisting(Location location) {
++ StoreConnection sConn = cache.get(location) ;
++ return sConn ;
++ }
++
++ public synchronized static boolean isSetup(Location location) {
++ return cache.containsKey(location) ;
++ }
++
++ /**
++ * Return a {@code StoreConnection} for a particular location,
++ * creating it if it does not exist in storage.
++ */
++ private synchronized static StoreConnection make(Location location, StoreParams params) {
++ StoreConnection sConn = cache.get(location) ;
++ if ( sConn == null ) {
++ ProcessFileLock lock = null;
++ if (SystemTDB.DiskLocationMultiJvmUsagePrevention && ! location.isMem() ) {
++ lock = lockForLocation(location);
++ // Take the lock. This is atomic.
++ lock.lockEx();
++ }
++
++ // Recovery happens when TransactionCoordinator.start is called
++ // during the building of the DatasetGraphTxn.
++
++ DatasetGraphTDB dsg = (DatasetGraphTDB)TDBBuilder.build(location, params) ;
++
++ sConn = new StoreConnection(dsg, lock) ;
++ if (!location.isMemUnique())
++ cache.put(location, sConn) ;
++ }
++ return sConn ;
++ }
++
++ /**
++ * Stop managing all locations.
++ * Use with extreme care.
++ * This is intended to support internal testing.
++ */
++ public static synchronized void internalReset() {
++ // Copy to avoid potential CME.
++ Set<Location> x = new HashSet<>(cache.keySet()) ;
++ for (Location loc : x)
++ internalExpel(loc, true) ;
++ cache.clear() ;
++ ChannelManager.reset();
++ }
++
++ /** Stop managing a location. There should be no transactions running. */
++ public static synchronized void release(Location location) {
++ internalExpel(location, false) ;
++ }
++
++ /** Use via {@link TDBInternal#expel} wherever possible.
++ * <p>
++ * Stop managing a location.<br/>
++ * Use with great care (testing only).
++ */
++ public static synchronized void internalExpel(Location location, boolean force) {
++ StoreConnection sConn = cache.get(location) ;
++ if (sConn == null) return ;
++
++ TransactionCoordinator txnCoord = sConn.getDatasetGraphTDB().getTxnSystem().getTxnMgr();
++ if (!force && txnCoord.countActive() > 0 )
++ throw new TransactionException("Can't expel: Active transactions for location: " + location) ;
++
++ // No transactions at this point
++ // (or we don't care and are clearing up forcefully.)
++
++ sConn.getDatasetGraphTDB().shutdown() ;
++ // Done by DatasetGraphTDB()
++ //txnCoord.shutdown();
++
++ sConn.isValid = false ;
++ cache.remove(location) ;
++
++ // Release the lock after the cache is emptied.
++ if (SystemTDB.DiskLocationMultiJvmUsagePrevention && ! location.isMem() ) {
++ if ( ! sConn.lock.isLockedHere() )
++ SystemTDB.errlog.warn("Location " + location.getDirectoryPath() + " was not locked by this process.");
++ sConn.lock.unlock();
++ ProcessFileLock.release(sConn.lock);
++ }
++ }
++
++ /** Create or fetch a {@link ProcessFileLock} for a Location */
++ public static ProcessFileLock lockForLocation(Location location) {
++ FileOps.ensureDir(location.getDirectoryPath());
++ String lockFilename = location.getPath(Names.TDB_LOCK_FILE);
++ Path path = Paths.get(lockFilename);
++ try {
++ path.toFile().createNewFile();
++ } catch(IOException ex) { IO.exception(ex); return null; }
++ return ProcessFileLock.create(lockFilename);
++ }
++
++ private final DatasetGraphTDB datasetGraph ;
++ // This is the location of the database itself, not the TDB2 container directory.
++ private final Location location ;
++ private final ProcessFileLock lock ;
++ private boolean isValid = true ;
++ private volatile boolean haveUsedInTransaction = false ;
++
++ private StoreConnection(DatasetGraphTDB dsg, ProcessFileLock fileLock)
++ {
++ this.datasetGraph = dsg ;
++ this.location = dsg.getLocation() ;
++ this.lock = fileLock;
++ }
++
++ public DatasetGraph getDatasetGraph() {
++ return datasetGraph;
++ }
++
++ public DatasetGraphTDB getDatasetGraphTDB() {
++ return datasetGraph;
++ }
++
++ public Location getLocation() {
++ return location ;
++ }
++
++ public ProcessFileLock getLock() {
++ return lock ;
++ }
++
++}
++
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/SystemTDB.java
----------------------------------------------------------------------
diff --cc jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/SystemTDB.java
index 0000000,0000000..51ef995
new file mode 100644
--- /dev/null
+++ b/jena-db/tdb2/src/main/java/org/seaborne/tdb2/sys/SystemTDB.java
@@@ -1,0 -1,0 +1,416 @@@
++/*
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ *
++ * See the NOTICE file distributed with this work for additional
++ * information regarding copyright ownership.
++ */
++
++package org.seaborne.tdb2.sys;
++
++import java.io.FileNotFoundException ;
++import java.io.IOException ;
++import java.util.Properties ;
++
++import org.apache.jena.atlas.io.IO ;
++import org.apache.jena.atlas.lib.PropertyUtils ;
++import org.apache.jena.atlas.logging.Log ;
++import org.apache.jena.query.ARQ ;
++import org.apache.jena.sparql.engine.optimizer.reorder.ReorderLib ;
++import org.apache.jena.sparql.engine.optimizer.reorder.ReorderTransformation ;
++import org.apache.jena.sparql.util.Symbol ;
++import org.apache.jena.system.JenaSystem ;
++import org.seaborne.dboe.base.block.FileMode ;
++import org.seaborne.dboe.base.file.ProcessFileLock;
++import org.seaborne.dboe.base.record.RecordFactory ;
++import org.seaborne.dboe.sys.Sys;
++import org.seaborne.tdb2.TDB2 ;
++import org.seaborne.tdb2.TDBException ;
++import org.seaborne.tdb2.store.NodeId ;
++import org.slf4j.Logger ;
++import org.slf4j.LoggerFactory ;
++
++public class SystemTDB
++{
++ static { JenaSystem.init(); }
++
++ // NB Same logger as the TDB class because this class is the system info but kept out of TDB javadoc.
++ // It's visibility is TDB, not really public.
++ private static final Logger log = LoggerFactory.getLogger(TDB2.class) ;
++
++ /** TDB System log - use for general messages (a few) and warnings.
++ * Generally, do not log events unless you want every user to see them every time.
++ * TDB is an embedded database - libraries and embedded systems should be seen and not heard.
++ * @see #errlog
++ */
++ // This was added quite late in TDB so need to check it's used appropriately - check for Log.*
++ public static final Logger syslog = LoggerFactory.getLogger("TDB") ;
++ /** Send warnings and error */
++ public static final Logger errlog = LoggerFactory.getLogger("TDB") ;
++
++ // ---- Constants that can't be changed without invalidating on-disk data.
++
++// /** Size, in bytes, of a Java long */
++// public static final int SizeOfLong = Long.SIZE/Byte.SIZE ;
++//
++// /** Size, in bytes, of a Java int */
++// public static final int SizeOfInt = Integer.SIZE/Byte.SIZE ;
++
++ /** Size, in bytes, of the persistent representation of a node id */
++ public static final int SizeOfNodeId = NodeId.SIZE ;
++
++ /** Size, in bytes, of a pointer between blocks */
++ public static final int SizeOfPointer = Sys.SizeOfInt ;
++
++ // ---- Node table related
++
++ /** Size, in bytes, of a triple index record. */
++ public static final int LenIndexTripleRecord = 3 * NodeId.SIZE ;
++ /** Size, in bytes, of a quad index record. */
++ public static final int LenIndexQuadRecord = 4 * NodeId.SIZE ;
++
++ /** Size, in bytes, of a Node hash.
++ * In TDB 0.7.X and before this was 8 bytes (64/8).
++ * In TDB 0.8.0 and above it is 16 bytes (128/8).
++ * These two systems are not compatible.
++ */
++ //public static final int LenNodeHash = SizeOfLong ; // TDB <= 0.7.X
++ public static final int LenNodeHash = 128/8 ; // TDB >= 0.8.0
++
++ // ---- Symbols and similar
++
++ // ---- Record factories
++ public final static RecordFactory indexRecordTripleFactory = new RecordFactory(LenIndexTripleRecord, 0) ;
++ public final static RecordFactory indexRecordQuadFactory = new RecordFactory(LenIndexQuadRecord, 0) ;
++ public final static RecordFactory nodeRecordFactory = new RecordFactory(LenNodeHash, SizeOfNodeId) ;
++
++ // Test show no visable effect.
++// /** Unit of flushing the data when loading - data phase. -1 means off, sync at end of load only. */
++// public final static long LoadFlushTickPrimary = -1 ; // 10*1000*1000 ;
++// /** Unit of flushing the data when loading - index phase -1 means off, sync at end of load only. */
++// public final static long LoadFlushTickSecondary = -1 ; // 10*1000*1000 ;
++
++ /** Root of TDB-defined parameter names */
++ public static final String symbolNamespace = "http://jena.hpl.hp.com/TDB#" ;
++
++ /** Root of TDB-defined parameter short names */
++ public static final String tdbSymbolPrefix = "tdb" ;
++
++ /** Root of any TDB-defined Java system properties */
++ public static final String tdbPropertyRoot = "org.apache.jena.tdb" ;
++
++ /** Log duplicates during loading */
++ public static final Symbol symLogDuplicates = allocSymbol("logDuplicates") ;
++
++ /** File mode : one of "direct", "mapped", "default" */
++ public static final Symbol symFileMode = allocSymbol("fileMode") ;
++
++ /** Index type */
++ public static final Symbol symIndexType = allocSymbol("indexType") ;
++
++ /** Experimental : triple and quad filtering at scan level */
++ public static final Symbol symTupleFilter = allocSymbol("tupleFilter") ;
++
++ private static final String propertyFileKey1 = tdbPropertyRoot+".settings" ;
++ private static final String propertyFileKey2 = tdbSymbolPrefix+":settings" ;
++
++ private static String propertyFileName = null ;
++ static {
++ propertyFileName = System.getProperty(propertyFileKey1) ;
++ if ( propertyFileName == null )
++ propertyFileName = System.getProperty(propertyFileKey2) ;
++ }
++
++ public static final boolean is64bitSystem = determineIf64Bit() ;
++
++ private static Properties properties = readPropertiesFile() ;
++
++ // To make the class initialize
++ static public void init() {}
++
++ /** Size, in bytes, of a block */
++ public static final int BlockSize = 8*1024 ; // intValue("BlockSize", 8*1024) ;
++
++ /** Size, in bytes, of a block for testing */
++ public static final int BlockSizeTest = 1024 ; // intValue("BlockSizeTest", 1024) ;
++
++ /** Size, in bytes, of a block for testing */
++ public static final int BlockSizeTestMem = 500 ;
++
++// /** Size, in bytes, of a memory block */
++// public static final int BlockSizeMem = 32*8 ; //intValue("BlockSizeMem", 32*8 ) ;
++
++ /** order of an in-memory BTree or B+Tree */
++ public static final int OrderMem = 5 ; // intValue("OrderMem", 5) ;
++
++ /** Size, in bytes, of a segment (used for memory mapped files) */
++ public static final int SegmentSize = 8*1024*1024 ; // intValue("SegmentSize", 8*1024*1024) ;
++
++ // ---- Cache sizes (within the JVM)
++
++ public static final int ObjectFileWriteCacheSize = 8*1024 ;
++
++ /** Size of Node to NodeId cache.
++ * Used to map from Node to NodeId spaces.
++ * Used for loading and for query preparation.
++ */
++ public static final int Node2NodeIdCacheSize = intValue("Node2NodeIdCacheSize", ( is64bitSystem ? 200*1000 : 20*1000 )) ;
++
++ /** Size of NodeId to Node cache.
++ * Used to map from NodeId to Node spaces.
++ * Used for retriveing results.
++ */
++ public static final int NodeId2NodeCacheSize = intValue("NodeId2NodeCacheSize", ( is64bitSystem ? 750*1000 : 20*1000 ) ) ;
++
++ /** Size of Node lookup miss cache. */
++ public static final int NodeMissCacheSize = 1000 ;
++
++ /** Size of the delayed-write block cache (32 bit systems only). Per file. */
++ public static final int BlockWriteCacheSize = intValue("BlockWriteCacheSize", 1000) ;
++
++ /** Size of read block cache (32 bit systems only). Per file. */
++ public static final int BlockReadCacheSize = intValue("BlockReadCacheSize", 5*1000) ;
++
++ // ---- Misc
++
++// /** Number of adds/deletes between calls to sync (-ve to disable) */
++// public static final int SyncTick = intValue("SyncTick", -1) ;
++
++ /** Default BGP optimizer */
++ public static ReorderTransformation defaultReorderTransform = ReorderLib.fixed() ;
++
++ /** Unsupported (for non-standard setups)
++ * @see #enableInlineLiterals
++ */
++ private static String propertyEnableInlineLiterals1 = "org.apache.jena.tdb.store.enableInlineLiterals" ;
++ /** Unsupported (for non-standard setups)
++ * @see #enableInlineLiterals
++ */
++ private static String propertyEnableInlineLiterals2 = "tdb:store.enableInlineLiterals" ;
++ /** <b>Unsupported</b> (for non-standard setups).
++ * This controls whether literal values are inlined into NodeIds.
++ * This is a major efficiency boost and is the default setting.
++ * It can be set false with {@code -Dtdb:store.enableInlineLiterals=false}.
++ * Do not mix databases created with this set to different values.
++ * Chaos and incorrect results will result.
++ * Use with care. No support.
++ * Default setting is {@code true}
++ */
++ public static final boolean enableInlineLiterals ;
++ static { // Set enableInlineLiterals from system properties.
++ Properties sysProperties = System.getProperties() ;
++ String key = null ;
++ if ( sysProperties.containsKey(propertyEnableInlineLiterals1) )
++ key = propertyFileKey1 ;
++ else if ( sysProperties.containsKey(propertyEnableInlineLiterals2) )
++ key = propertyFileKey2 ;
++ if ( key == null )
++ enableInlineLiterals = true ; // Normal value.
++ else
++ enableInlineLiterals = Boolean.valueOf(sysProperties.getProperty(key)) ;
++ }
++
++// public static void setNullOut(boolean nullOut)
++// { SystemTDB.NullOut = nullOut ; }
++//
++// /** Are we nulling out unused space in bytebuffers (records, points etc) */
++// public static boolean getNullOut()
++// { return SystemTDB.NullOut ; }
++
++ /** null out (with the FillByte) freed up space in buffers */
++ public static boolean NullOut = false ;
++
++ /** FillByte value for NullOut */
++ public static final byte FillByte = (byte)0xFF ;
++
++ public static boolean Checking = false ; // This isn't used enough!
++
++ /**
++ * When enabled, a {@link ProcessFileLock} is used to block over rpcoesses oening this database.
++ */
++ public static boolean DiskLocationMultiJvmUsagePrevention = true;
++
++ public static void panic(Class<?> clazz, String string)
++ {
++ Log.error(clazz, string) ;
++ throw new TDBException(string) ;
++ }
++
++ public static Symbol allocSymbol(String shortName)
++ {
++ if ( shortName.startsWith(SystemTDB.tdbSymbolPrefix))
++ throw new TDBException("Symbol short name begins with the TDB namespace prefix: "+shortName) ;
++ if ( shortName.startsWith("http:"))
++ throw new TDBException("Symbol short name begins with http: "+shortName) ;
++ return allocSymbol(SystemTDB.symbolNamespace, shortName) ;
++ }
++
++ private static Symbol allocSymbol(String namespace, String shortName)
++ {
++ return Symbol.create(namespace+shortName) ;
++ }
++
++ // ----
++
++ private static int intValue(String prefix, String name, int defaultValue)
++ {
++ if ( ! prefix.endsWith(".") )
++ name = prefix+"."+name ;
++ else
++ name = prefix+name ;
++ return intValue(name, defaultValue) ;
++ }
++
++ private static int intValue(String name, int defaultValue)
++ {
++ if ( name == null ) return defaultValue ;
++ if ( name.length() == 0 ) throw new TDBException("Empty string for value name") ;
++
++ if ( properties == null )
++ return defaultValue ;
++
++ String x = properties.getProperty(name) ;
++ if ( x == null )
++ return defaultValue ;
++ TDB2.logInfo.info("Set: "+name+" = "+x) ;
++ int v = Integer.parseInt(x) ;
++ return v ;
++ }
++
++ private static Properties readPropertiesFile()
++ {
++ if ( propertyFileName == null )
++ return null ;
++
++ Properties p = new Properties() ;
++ try
++ {
++ TDB2.logInfo.info("Using properties from '"+propertyFileName+"'") ;
++ PropertyUtils.loadFromFile(p, propertyFileName) ;
++ } catch (FileNotFoundException ex)
++ {
++ log.debug("No system properties file ("+propertyFileName+")") ;
++ return null ;
++ } catch (IOException ex) { IO.exception(ex) ; }
++ return p ;
++ }
++
++ // --------
++
++ public static final boolean isWindows = determineIfWindows() ; // Memory mapped files behave differently.
++
++ //Or look in File.listRoots.
++ //Alternative method:
++ // http://stackoverflow.com/questions/1293533/name-of-the-operating-system-in-java-not-os-name
++
++ private static boolean determineIfWindows() {
++ String s = System.getProperty("os.name") ;
++ if ( s == null )
++ return false ;
++ return s.startsWith("Windows ") ;
++ }
++
++ private static boolean determineIf64Bit()
++ {
++ String s = System.getProperty("sun.arch.data.model") ;
++ if ( s != null )
++ {
++ boolean b = s.equals("64") ;
++ TDB2.logInfo.debug("System architecture: "+(b?"64 bit":"32 bit")) ;
++ return b ;
++ }
++ // Not a SUN VM
++ s = System.getProperty("java.vm.info") ;
++ if ( s == null )
++ {
++ log.warn("Can't determine the data model") ;
++ return false ;
++ }
++ log.debug("Can't determine the data model from 'sun.arch.data.model' - using java.vm.info") ;
++ boolean b = s.contains("64") ;
++ TDB2.logInfo.debug("System architecture: (from java.vm.info) "+(b?"64 bit":"32 bit")) ;
++ return b ;
++ }
++
++ // ---- File mode
++
++ private static FileMode fileMode = null ;
++ public static FileMode fileMode()
++ {
++ if ( fileMode == null )
++ fileMode = determineFileMode() ;
++ return fileMode ;
++ }
++
++ public static void setFileMode(FileMode newFileMode)
++ {
++ if ( fileMode != null )
++ {
++ log.warn("System file mode already determined - setting it has no effect") ;
++ return ;
++ }
++ fileMode = newFileMode ;
++ }
++
++ // So the test suite can setup thing up ... very carefully.
++ /*package*/ static void internalSetFileMode(FileMode newFileMode)
++ {
++ fileMode = newFileMode ;
++ }
++
++ private static FileMode determineFileMode()
++ {
++ // Be careful that this is not called very, very early, before --set might be seen.
++ // Hence delayed access above in fileMode().
++
++ String x = ARQ.getContext().getAsString(SystemTDB.symFileMode, "default") ;
++
++ if ( x.equalsIgnoreCase("direct") )
++ {
++ TDB2.logInfo.info("File mode: direct (forced)") ;
++ return FileMode.direct ;
++ }
++ if ( x.equalsIgnoreCase("mapped") )
++ {
++ TDB2.logInfo.info("File mode: mapped (forced)") ;
++ return FileMode.mapped ;
++ }
++
++ if ( x.equalsIgnoreCase("default") )
++ {
++ if ( is64bitSystem )
++ {
++ TDB2.logInfo.debug("File mode: Mapped") ;
++ return FileMode.mapped ;
++ }
++ TDB2.logInfo.debug("File mode: Direct") ;
++ return FileMode.direct ;
++ }
++ throw new TDBException("Unrecognized file mode (not one of 'default', 'direct' or 'mapped': "+x) ;
++ }
++
++// public static Dataset setNonTransactional(Dataset dataset) {
++// if ( dataset.isInTransaction() )
++// return dataset ; // And hope it's a write transaction.
++// dataset.begin(ReadWrite.WRITE);
++// //Or wrap DatasetGraphTDB?
++// return dataset ;
++// }
++//
++// public static DatasetGraph setNonTransactional(DatasetGraph dataset) {
++// if ( dataset.isInTransaction() )
++// return dataset ;
++// dataset.begin(ReadWrite.WRITE);
++// return dataset ;
++// }
++
++}