You are viewing a plain text version of this content. The canonical link for it is here.
Posted to pr@jena.apache.org by GitBox <gi...@apache.org> on 2020/08/16 17:21:19 UTC
[GitHub] [jena] afs commented on a change in pull request #780: JENA-1929: Detect and use TDB database type for existing database.
afs commented on a change in pull request #780:
URL: https://github.com/apache/jena/pull/780#discussion_r471136889
##########
File path: jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/system/spot/SpotTDB2.java
##########
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.fuseki.system.spot;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.jena.atlas.io.IO;
+import org.apache.jena.dboe.base.file.Location;
+import org.apache.jena.dboe.sys.Names;
+import org.apache.jena.tdb2.TDBException;
+import org.apache.jena.tdb2.params.StoreParams;
+import org.apache.jena.tdb2.params.StoreParamsCodec;
+import org.apache.jena.tdb2.sys.DatabaseOps;
+import org.apache.jena.tdb2.sys.IOX;
+import org.apache.jena.tdb2.sys.Util;
+
+class SpotTDB2 {
+
+ /* TDB2 layout
+ * top:
+ *
+ * Data-0001/ tdb.lock
+ *
+ * Data-0001:
+ * GOSP.bpt GPU.dat nodes.dat OSPG.dat POSG.idn SPO.bpt
+ * GOSP.dat GPU.idn nodes-data.bdf OSPG.idn POS.idn SPO.dat
+ * GOSP.idn GSPO.bpt nodes-data.obj OSP.idn prefixes.bpt SPOG.bpt
+ * GPOS.bpt GSPO.dat nodes.idn POS.bpt prefixes.dat SPOG.dat
+ * GPOS.dat GSPO.idn OSP.bpt POS.dat prefixes-data.bdf SPOG.idn
+ * GPOS.idn journal.jrnl OSP.dat POSG.bpt prefixes-data.obj SPO.idn
+ * GPU.bpt nodes.bpt OSPG.bpt POSG.dat prefixes.idn tdb.lock
+ */
+//
+// // TEMP
+// // These are in DatabaseOps
+// private static final String dbPrefix = "Data";
+// private static final String SEP = "-";
+
+ public static boolean isTDB2(String pathname) {
+ return isTDB2(Location.create(pathname));
+ }
+
+ /**
+ * Test to see is a location is either empty (and a fresh TDB2 database can be
+ * created there) or has looks like it is an existing TDB2 database.
+ * See {@link #checkTDB2(Location)} for a test that the location is a valid, existing
+ * database.
+ */
+ public static boolean isTDB2(Location location) {
+ if ( location.isMem() )
+ return true;
+ if ( ! location.exists() )
+ return false;
+ if ( isEmpty(location) )
+ return true;
+ // Look for Data-*
+ Path db = storageDir(location);
+ if ( db == null )
+ // Uninitialized?
+ // Or TDB1?
+ return ! SpotTDB1.isTDB1(location.getDirectoryPath());
+ // Validate storage.
+ Location storageLocation = IOX.asLocation(db);
+ return isTDB2_Storage(storageLocation);
+ }
+
+ /** Quick check for TDB2. Does not check for validity. */
+ private static boolean isTDB2_Storage(Location location) {
+ // Journal check.
+ //return exists(location, Names.journalFileBase, Names.extJournal);
+ // Test for triples primary index
+ StoreParams params = getStoreParams(location);
+ return exists(location, params.getPrimaryIndexTriples(),
+ Names.extBptTree, Names.extBptRecords, Names.extBptState);
+ }
+
+ public static void checkTDB2(String pathname) {
+ checkTDB2(Location.create(pathname));
+ }
+
+ public static void checkTDB2(Location location) {
+ if ( location.isMem() )
+ return;
+ if ( isEmpty(location) )
+ return;
+ Path db = storageDir(location);
+ if ( db == null )
+ // Uninitialized
+ return ;
+ // Storage. Easier to work in "Location".
+ Location locStorage = Location.create(db);
+ checkStorageArea(locStorage);
+ }
+
+ /** Return the current active database area within a database directory. */
+ private static Path storageDir(Location location) {
+ // Database directory
+ Path path = IOX.asPath(location);
+ // Storage directory in database directory.
+ Path db = findLocation(path, DatabaseOps.dbPrefix);
+ return db;
+ }
+
+ private static Path findLocation(Path directory, String namebase) {
+ if ( ! Files.exists(directory) )
+ return null;
+ // In-order, low to high.
+ List<Path> maybe = IOX.scanForDirByPattern(directory, namebase, DatabaseOps.SEP);
+ return Util.getLastOrNull(maybe);
+ }
+
+ // Places for StoreParams: location or default
+ private static StoreParams getStoreParams(Location location) {
+ StoreParams params = StoreParamsCodec.read(location);
+ if ( params == null )
+ params = StoreParams.getDftStoreParams();
+ return params;
+ }
+
+ /**
+ * Check all files exist for a TDB2 database, or the area is empty (and so a new
+ * database can be created in the location). Throw {@link TDBException} is a file
+ * is missing.
+ */
+ private static void checkStorageArea(Location location) {
+
+ if ( location.isMem() )
+ return;
+
+ if ( isEmpty(location) )
+ return;
+
+ // Journal, fixed name.
+ validate(location, Names.journalFileBase, Names.extJournal);
+
+ // Places for StoreParams: location or default
+ StoreParams params = getStoreParams(location);
+
+ validate(location, Names.journalFileBase, Names.extJournal);
Review comment:
Fixed.
##########
File path: jena-fuseki2/jena-fuseki-core/src/main/java/org/apache/jena/fuseki/system/spot/SpotTDB1.java
##########
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.fuseki.system.spot;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.jena.atlas.io.IO;
+import org.apache.jena.tdb.TDBException;
+import org.apache.jena.tdb.base.file.Location;
+import org.apache.jena.tdb.setup.StoreParams;
+import org.apache.jena.tdb.setup.StoreParamsCodec;
+import org.apache.jena.tdb.sys.Names;
+
+class SpotTDB1 {
+ /* TDB1 layout
+ *
+ * GOSP.dat GSPO.dat node2id.idn OSPG.idn POSG.idn prefixes.dat SPOG.dat
+ * GOSP.idn GSPO.idn nodes.dat OSP.idn POS.idn prefixIdx.dat SPOG.idn
+ * GPOS.dat journal.jrnl OSP.dat POS.dat prefix2id.dat prefixIdx.idn SPO.idn
+ * GPOS.idn node2id.dat OSPG.dat POSG.dat prefix2id.idn SPO.dat
+ */
+
+ /* StoreParams
+ * primaryIndexTriples dft:SPO
+ * tripleIndexes dft:[SPO, POS, OSP]
+ *
+ * primaryIndexQuads dft:GSPO
+ * quadIndexes dft:[GSPO, GPOS, GOSP, POSG, OSPG, SPOG]
+ *
+ * primaryIndexPrefix dft:GPU
+ * prefixIndexes dft:[GPU]
+ * -- Actual name of GPU index.
+ * indexPrefix dft:prefixIdx BPT
+ *
+ * -- NodeTable
+ * indexNode2Id dft:node2id
+ * indexId2Node dft:nodes
+ *
+ * -- NodeTable
+ * prefixNode2Id dft:prefix2id BPT *
+ * prefixId2Node dft:prefixes DAT *
+ */
+
+ /**
+ * Test to see is a location is either empty (and a fresh TDB1 database can be
+ * created there) or has looks like it is an existing TDB1 database. See
+ * {@link #checkTDB1(Location)} for a test that the location is a valid, existing
+ * database.
+ */
+ public static boolean isTDB1(String pathname) {
+ return isTDB1(Location.create(pathname));
+ }
+
+ /**
+ * Test to see is a location is either empty (and a fresh TDB1 database can be
+ * created there) or has looks like it is an existing TDB1 database. See
+ * {@link #checkTDB1(Location)} for a test that the location is a valid, existing
+ * database.
+ */
+ public static boolean isTDB1(Location location) {
+ if ( location.isMem() )
+ return true;
+
+ if ( ! location.exists() )
+ return false;
+ if ( isEmpty(location) )
+ return true;
+
+ // Very occasionally, deleting the journal makes sense.
+
+ return isTDB1_Storage(location);
+ }
+
+ /** Quick check for TDB1. Does not check for validity. */
+ private static boolean isTDB1_Storage(Location location) {
+ // Has a journal.
+ //return exists(location, Names.journalFileBase, Names.extJournal);
+ // Test for triples primary index
+ StoreParams params = getStoreParams(location);
+ return exists(location, params.getPrimaryIndexTriples(), Names.bptExtTree, Names.bptExtRecords);
+ }
+
+ /**
+ * Check all files exist for a TDB1 database, or the area is empty (and so a new
+ * database can be created in the location). Throw an exception if a file
+ * is missing.
+ */
+ public static void checkTDB1(String pathname) {
+ checkTDB1(Location.create(pathname));
+ }
+
+ /**
+ * Check all files exist for a TDB1 database, or the area is empty (and so a new
+ * database can be created in the location). Throw an exception if a file
+ * is missing.
+ */
+ public static void checkTDB1(Location location) {
+ if ( location.isMem() )
+ return;
+
+ if ( isEmpty(location) )
+ return;
+
+ // Journal, fixed name.
+ validate(location, Names.journalFileBase, Names.extJournal);
+
+ if ( ! isTDB1(location) )
+ throw new TDBException("Not a TDB1 location: "+location);
+
+ // Places for StoreParams: location or default
+ StoreParams params = getStoreParams(location);
+
+ validate(location, Names.journalFileBase, Names.extJournal);
Review comment:
Fixed
##########
File path: jena-fuseki2/jena-fuseki-webapp/src/main/java/org/apache/jena/fuseki/cmd/DSGSetup.java
##########
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.fuseki.cmd;
+
+import jena.cmd.CmdException;
+import org.apache.jena.atlas.io.IO;
+import org.apache.jena.fuseki.mgt.Template;
+import org.apache.jena.fuseki.system.spot.TDBOps;
+
+public class DSGSetup {
Review comment:
Good point. renamed one and also made them package scoped.
##########
File path: jena-fuseki2/jena-fuseki-webapp/src/main/java/org/apache/jena/fuseki/webapp/FusekiWebapp.java
##########
@@ -284,13 +287,15 @@ private static DataAccessPoint configFromTemplate(String templateFile, String da
Fuseki.configLog.info("Template file: " + templateFile);
String dir = params.get(Template.DIR);
if ( dir != null ) {
- if ( Objects.equals(dir, Names.memName) ) {
- Fuseki.configLog.info("TDB dataset: in-memory");
- } else {
- if ( !FileOps.exists(dir) )
- throw new CmdException("Directory not found: " + dir);
- Fuseki.configLog.info("TDB dataset: directory=" + dir);
- }
+ if ( ! Objects.equals(dir, Names.memName) && !FileOps.exists(dir) )
+ throw new CmdException("Directory not found: " + dir);
+// if ( Objects.equals(dir, Names.memName) ) {
+// Fuseki.configLog.info("TDB dataset: in-memory");
+// } else {
+// if ( !FileOps.exists(dir) )
+// throw new CmdException("Directory not found: " + dir);
+// Fuseki.configLog.info("TDB dataset: directory=" + dir);
+// }
Review comment:
Done!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: pr-unsubscribe@jena.apache.org
For additional commands, e-mail: pr-help@jena.apache.org