You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/09/30 18:45:25 UTC
[48/50] [abbrv] git commit: implemented a collection of different id
generators (MARMOTTA-324)
implemented a collection of different id generators (MARMOTTA-324)
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/4b8b8680
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/4b8b8680
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/4b8b8680
Branch: refs/heads/master
Commit: 4b8b86802a9814f8cffbb03a5262b00f7a42f9a9
Parents: c576351
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Mon Sep 30 18:12:32 2013 +0200
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Mon Sep 30 18:12:32 2013 +0200
----------------------------------------------------------------------
libraries/kiwi/kiwi-triplestore/pom.xml | 6 +
.../marmotta/kiwi/config/KiWiConfiguration.java | 40 ++---
.../generator/DatabaseSequenceIDGenerator.java | 75 ++++++++
.../marmotta/kiwi/generator/IDGenerator.java | 61 +++++++
.../kiwi/generator/IDGeneratorType.java | 54 ++++++
.../generator/MemorySequenceIDGenerator.java | 178 +++++++++++++++++++
.../kiwi/generator/SnowflakeIDGenerator.java | 178 +++++++++++++++++++
.../kiwi/generator/UUIDRandomIDGenerator.java | 82 +++++++++
.../kiwi/generator/UUIDTimeIDGenerator.java | 108 +++++++++++
.../kiwi/persistence/KiWiConnection.java | 92 +++-------
.../kiwi/persistence/KiWiPersistence.java | 135 +++-----------
parent/pom.xml | 13 ++
.../services/triplestore/SesameServiceImpl.java | 29 ++-
.../main/resources/config-defaults.properties | 5 +-
.../resources/config-descriptions.properties | 9 +-
15 files changed, 843 insertions(+), 222 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/pom.xml
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/pom.xml b/libraries/kiwi/kiwi-triplestore/pom.xml
index b8e880a..956ae68 100644
--- a/libraries/kiwi/kiwi-triplestore/pom.xml
+++ b/libraries/kiwi/kiwi-triplestore/pom.xml
@@ -72,6 +72,12 @@
<artifactId>tomcat-jdbc</artifactId>
</dependency>
+ <!-- primary key generation -->
+ <dependency>
+ <groupId>com.fasterxml.uuid</groupId>
+ <artifactId>java-uuid-generator</artifactId>
+ </dependency>
+
<!-- Sesame dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
index 6767100..3daf16d 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/config/KiWiConfiguration.java
@@ -17,6 +17,7 @@
*/
package org.apache.marmotta.kiwi.config;
+import org.apache.marmotta.kiwi.generator.IDGeneratorType;
import org.apache.marmotta.kiwi.persistence.KiWiDialect;
/**
@@ -83,13 +84,10 @@ public class KiWiConfiguration {
private int cursorSize = 1000;
/**
- * If enabled, and batchCommit is also true, load sequence values into static memory fields once and increment
- * values purely in-memory. The last value is then written back on batch commits.
+ * The method to use for generating row IDs. Starting with Marmotta 3.2, the default is to use the Twitter Snowflake
+ * algorithm.
*/
- private boolean memorySequences = true;
-
-
- private boolean commitSequencesOnCommit = true;
+ private IDGeneratorType idGeneratorType = IDGeneratorType.SNOWFLAKE;
public KiWiConfiguration(String name, String jdbcUrl, String dbUser, String dbPassword, KiWiDialect dialect) {
@@ -201,32 +199,16 @@ public class KiWiConfiguration {
this.cursorSize = cursorSize;
}
- public boolean isMemorySequences() {
- return memorySequences;
- }
-
/**
- * Enable in-memory sequences. If enabled, and batchCommit is also true, load sequence values into static memory
- * fields once and increment values purely in-memory. The last value is then written back on batch commits. This
- * feature can avoid many database accesses and connections and therefore give significant performance improvements.
- * (EXPERIMENTAL).
+ * The ID generator used for generating row IDs in the database. See documentation in {@link IDGeneratorType}
+ *
+ * @return the type defined for this configuration
*/
- public void setMemorySequences(boolean memorySequences) {
- this.memorySequences = memorySequences;
+ public IDGeneratorType getIdGeneratorType() {
+ return idGeneratorType;
}
-
- public boolean isCommitSequencesOnCommit() {
- return commitSequencesOnCommit;
- }
-
- /**
- * This flag determines whether memory sequences should be stored back into the database on every commit or only
- * when the repository shuts down. Saving back on every commit is safer, but has less performance.
- *
- * @param commitSequencesOnCommit
- */
- public void setCommitSequencesOnCommit(boolean commitSequencesOnCommit) {
- this.commitSequencesOnCommit = commitSequencesOnCommit;
+ public void setIdGeneratorType(IDGeneratorType idGeneratorType) {
+ this.idGeneratorType = idGeneratorType;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/DatabaseSequenceIDGenerator.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/DatabaseSequenceIDGenerator.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/DatabaseSequenceIDGenerator.java
new file mode 100644
index 0000000..ddaa094
--- /dev/null
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/DatabaseSequenceIDGenerator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.kiwi.generator;
+
+import org.apache.marmotta.kiwi.persistence.KiWiConnection;
+import org.apache.marmotta.kiwi.persistence.KiWiPersistence;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+ * Generate IDs using database sequences
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class DatabaseSequenceIDGenerator implements IDGenerator {
+
+ public DatabaseSequenceIDGenerator() {
+ }
+
+
+ /**
+ * Initialise the generator for the given persistence and module
+ */
+ @Override
+ public void init(KiWiPersistence persistence, String scriptName) {
+ }
+
+ /**
+ * Commit the current state of memory sequences to the database using the connection passed as second argument.
+ *
+ * @param persistence
+ * @param con
+ * @throws java.sql.SQLException
+ */
+ @Override
+ public void commit(KiWiPersistence persistence, Connection con) throws SQLException {
+ }
+
+ /**
+ * Shut down this id generator, performing any cleanups that might be necessary.
+ *
+ * @param persistence
+ */
+ @Override
+ public void shutdown(KiWiPersistence persistence) {
+
+ }
+
+ /**
+ * Return the next unique id for the type with the given name using the generator's id generation strategy.
+ *
+ * @param name
+ * @return
+ */
+ @Override
+ public long getId(String name, KiWiConnection connection) throws SQLException {
+ return connection.getDatabaseSequence(name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGenerator.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGenerator.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGenerator.java
new file mode 100644
index 0000000..2318a8d
--- /dev/null
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGenerator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.kiwi.generator;
+
+import org.apache.marmotta.kiwi.persistence.KiWiConnection;
+import org.apache.marmotta.kiwi.persistence.KiWiPersistence;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+ * An interface for database ID generators.
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public interface IDGenerator {
+
+ /**
+ * Return the next unique id for the type with the given name using the generator's id generation strategy.
+ * @param name
+ *
+ * @return
+ */
+ public long getId(String name, KiWiConnection connection) throws SQLException;
+
+ /**
+ * Initialise the generator for the given persistence and module
+ */
+ public void init(KiWiPersistence persistence, String scriptName);
+
+ /**
+ * Commit the current state of memory sequences to the database using the connection passed as second argument.
+ *
+ * @param persistence
+ * @param con
+ * @throws java.sql.SQLException
+ */
+ void commit(KiWiPersistence persistence, Connection con) throws SQLException;
+
+ /**
+ * Shut down this id generator, performing any cleanups that might be necessary.
+ *
+ * @param persistence
+ */
+ public void shutdown(KiWiPersistence persistence);
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGeneratorType.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGeneratorType.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGeneratorType.java
new file mode 100644
index 0000000..86bb6a2
--- /dev/null
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/IDGeneratorType.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.kiwi.generator;
+
+/**
+ * An enum for describing different ways of calculating the primary key ids for database rows in the KiWiConfiguration
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public enum IDGeneratorType {
+ /**
+ * Generate keys using database sequences (for databases which support it) or atomic table updates
+ */
+ DATABASE_SEQUENCE,
+
+ /**
+ * Load the sequences in-memory on startup and generate ids by incrementing an atomic counter. Much faster
+ * than database sequences and backwards-compatible, but only works for exclusive connections to the
+ * database (i.e. no other application is accessing the same database)
+ */
+ MEMORY_SEQUENCE,
+
+
+ /**
+ * Generate row ids using the least significant 8 bytes of time-based UUIDs as described in http://www.ietf.org/rfc/rfc4122.txt
+ */
+ UUID_TIME,
+
+
+ /**
+ * Generate the row ids using the least significant 8 bytes of secure random UUIDs as described in http://www.ietf.org/rfc/rfc4122.txt
+ */
+ UUID_RANDOM,
+
+ /**
+ * Generate the row ids using the Twitter Snowflake algorithm described in https://github.com/twitter/snowflake
+ */
+ SNOWFLAKE
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/MemorySequenceIDGenerator.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/MemorySequenceIDGenerator.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/MemorySequenceIDGenerator.java
new file mode 100644
index 0000000..908bc44
--- /dev/null
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/MemorySequenceIDGenerator.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.kiwi.generator;
+
+import com.google.common.util.concurrent.AtomicLongMap;
+import org.apache.marmotta.kiwi.persistence.KiWiConnection;
+import org.apache.marmotta.kiwi.persistence.KiWiPersistence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Generate unique IDs by incrementing an in-memory atomic long value for each sequence. This method is much faster
+ * than database sequences and backwards compatible (because values are written back to the database on each commit),
+ * but it does not work reliably if several applications access the same database.
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class MemorySequenceIDGenerator implements IDGenerator {
+ private static Logger log = LoggerFactory.getLogger(MemorySequenceIDGenerator.class);
+
+ /**
+ * A map holding in-memory sequences to be used for sequence caching in case the appropriate configuration option
+ * is configued and batched commits are enabled.
+ */
+ private AtomicLongMap<String> memorySequences;
+
+
+ // keep track which memory sequences have been updated and need to be written back
+ private Set<String> sequencesUpdated;
+
+ private ReentrantLock sequencesLock;
+
+
+
+ public MemorySequenceIDGenerator() {
+ this.sequencesLock = new ReentrantLock();
+ this.sequencesUpdated = new HashSet<>();
+ memorySequences = AtomicLongMap.create();
+ }
+
+ /**
+ * Initialise in-memory sequences if the feature is enabled.
+ */
+ @Override
+ public void init(KiWiPersistence persistence, String scriptName) {
+ sequencesLock.lock();
+ try {
+ try {
+ Connection con = persistence.getJDBCConnection(true);
+ try {
+ for(String sequenceName : persistence.getDialect().listSequences(scriptName)) {
+
+ // load sequence value from database
+ // if there is a preparation needed to update the transaction, run it first
+ if(persistence.getDialect().hasStatement(sequenceName+".prep")) {
+ PreparedStatement prepNodeId = con.prepareStatement(persistence.getDialect().getStatement(sequenceName+".prep"));
+ prepNodeId.executeUpdate();
+ prepNodeId.close();
+ }
+
+ PreparedStatement queryNodeId = con.prepareStatement(persistence.getDialect().getStatement(sequenceName));
+ ResultSet resultNodeId = queryNodeId.executeQuery();
+ try {
+ if(resultNodeId.next()) {
+ memorySequences.put(sequenceName,resultNodeId.getLong(1)-1);
+ } else {
+ throw new SQLException("the sequence did not return a new value");
+ }
+ } finally {
+ resultNodeId.close();
+ }
+
+ con.commit();
+ }
+ } finally {
+ persistence.releaseJDBCConnection(con);
+ }
+ } catch(SQLException ex) {
+ log.warn("database error: could not initialise in-memory sequences",ex);
+ }
+ } finally {
+ sequencesLock.unlock();
+ }
+
+ }
+
+ /**
+ * Commit the current state of memory sequences to the database using the connection passed as second argument.
+ *
+ * @param persistence
+ * @param con
+ * @throws SQLException
+ */
+ @Override
+ public void commit(KiWiPersistence persistence, Connection con) throws SQLException {
+ sequencesLock.lock();
+ try {
+ // clear existing list of updated sequences
+ Set<String> updated = this.sequencesUpdated;
+ this.sequencesUpdated = new HashSet<>();
+
+ try {
+ for(Map.Entry<String,Long> entry : memorySequences.asMap().entrySet()) {
+ if( updated.contains(entry.getKey()) && entry.getValue() > 0) {
+ PreparedStatement updateSequence = con.prepareStatement(persistence.getDialect().getStatement(entry.getKey()+".set"));
+ updateSequence.setLong(1, entry.getValue());
+ if(updateSequence.execute()) {
+ updateSequence.getResultSet().close();
+ } else {
+ updateSequence.getUpdateCount();
+ }
+ }
+ }
+ } catch(SQLException ex) {
+ // MySQL deadlock state, in this case we retry anyways
+ if(!"40001".equals(ex.getSQLState())) {
+ log.error("SQL exception:",ex);
+ }
+ throw ex;
+ }
+
+ } finally {
+ sequencesLock.unlock();
+ }
+ }
+
+ /**
+ * Shut down this id generator, performing any cleanups that might be necessary.
+ *
+ * @param persistence
+ */
+ @Override
+ public void shutdown(KiWiPersistence persistence) {
+
+ }
+
+ /**
+ * Return the next unique id for the type with the given name using the generator's id generation strategy.
+ *
+ * @param name
+ * @return
+ */
+ @Override
+ public long getId(String name, KiWiConnection connection) throws SQLException {
+ sequencesUpdated.add(name);
+
+ if(memorySequences != null) {
+ return memorySequences.incrementAndGet(name);
+ } else {
+ return 0;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/SnowflakeIDGenerator.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/SnowflakeIDGenerator.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/SnowflakeIDGenerator.java
new file mode 100644
index 0000000..a56d37d
--- /dev/null
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/SnowflakeIDGenerator.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.kiwi.generator;
+
+import org.apache.marmotta.kiwi.persistence.KiWiConnection;
+import org.apache.marmotta.kiwi.persistence.KiWiPersistence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Enumeration;
+import java.util.Random;
+
+/**
+ * Generate unique IDs using the Twitter Snowflake algorithm (see https://github.com/twitter/snowflake). Snowflake IDs
+ * are 64 bit positive longs composed of:
+ * - 41 bits time stamp
+ * - 10 bits machine id
+ * - 12 bits sequence number
+ *
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class SnowflakeIDGenerator implements IDGenerator {
+ private static Logger log = LoggerFactory.getLogger(SnowflakeIDGenerator.class);
+
+
+ private final long datacenterIdBits = 10L;
+ private final long maxDatacenterId = -1L ^ (-1L << datacenterIdBits);
+ private final long sequenceBits = 12L;
+
+ private final long datacenterIdShift = sequenceBits;
+ private final long timestampLeftShift = sequenceBits + datacenterIdBits;
+ private final long sequenceMask = -1L ^ (-1L << sequenceBits);
+
+ private final long twepoch = 1288834974657L;
+ private long datacenterId;
+
+ private volatile long lastTimestamp = -1L;
+ private volatile long sequence = 0L;
+
+
+ public SnowflakeIDGenerator() {
+ try {
+ datacenterId = getDatacenterId();
+ } catch (SocketException | UnknownHostException e) {
+ log.warn("SNOWFLAKE: could not determine machine address; using random datacenter ID");
+ Random rnd = new Random();
+ datacenterId = rnd.nextInt((int)maxDatacenterId) + 1;
+ }
+ if (datacenterId > maxDatacenterId || datacenterId < 0){
+ log.warn("SNOWFLAKE: datacenterId > maxDatacenterId; using random datacenter ID");
+ Random rnd = new Random();
+ datacenterId = rnd.nextInt((int)maxDatacenterId) + 1;
+ }
+
+ log.info("SNOWFLAKE: initialised with datacenter ID {}", datacenterId);
+ }
+
+ protected long tilNextMillis(long lastTimestamp){
+ long timestamp = System.currentTimeMillis();
+ while (timestamp <= lastTimestamp) {
+ timestamp = System.currentTimeMillis();
+ }
+ return timestamp;
+ }
+
+ protected long getDatacenterId() throws SocketException, UnknownHostException {
+ InetAddress ip = InetAddress.getLocalHost();
+ NetworkInterface network = null;
+
+
+ Enumeration<NetworkInterface> en = NetworkInterface.getNetworkInterfaces();
+ while (en.hasMoreElements()) {
+ NetworkInterface nint = en.nextElement();
+ if (!nint.isLoopback()) {
+ network = nint;
+ break;
+ }
+ }
+
+ byte[] mac = network.getHardwareAddress();
+
+ Random rnd = new Random();
+ byte rndByte = (byte)(rnd.nextInt() & 0x000000FF);
+
+ // take the last byte of the MAC address and a random byte as datacenter ID
+ long id = ((0x000000FF & (long)mac[mac.length-1]) | (0x0000FF00 & (((long)rndByte)<<8)))>>6;
+
+
+ return id;
+ }
+
+
+
+ /**
+ * Commit the current state of memory sequences to the database using the connection passed as second argument.
+ *
+ * @param persistence
+ * @param con
+ * @throws java.sql.SQLException
+ */
+ @Override
+ public void commit(KiWiPersistence persistence, Connection con) throws SQLException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ /**
+ * Return the next unique id for the type with the given name using the generator's id generation strategy.
+ *
+ * @param name
+ * @return
+ */
+ @Override
+ public synchronized long getId(String name, KiWiConnection connection) throws SQLException {
+ long timestamp = System.currentTimeMillis();
+ if(timestamp<lastTimestamp) {
+ log.warn("Clock moved backwards. Refusing to generate id for {} milliseconds.",(lastTimestamp - timestamp));
+ try {
+ Thread.sleep((lastTimestamp - timestamp));
+ } catch (InterruptedException e) {
+ }
+ }
+ if (lastTimestamp == timestamp) {
+ sequence = (sequence + 1) & sequenceMask;
+ if (sequence == 0) {
+ timestamp = tilNextMillis(lastTimestamp);
+ }
+ } else {
+ sequence = 0;
+ }
+ lastTimestamp = timestamp;
+ long id = ((timestamp - twepoch) << timestampLeftShift) | (datacenterId << datacenterIdShift) | sequence;
+
+ if(id < 0) {
+ log.warn("ID is smaller than 0: {}",id);
+ }
+ return id;
+ }
+
+ /**
+ * Initialise the generator for the given persistence and module
+ */
+ @Override
+ public void init(KiWiPersistence persistence, String scriptName) {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ /**
+ * Shut down this id generator, performing any cleanups that might be necessary.
+ *
+ * @param persistence
+ */
+ @Override
+ public void shutdown(KiWiPersistence persistence) {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDRandomIDGenerator.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDRandomIDGenerator.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDRandomIDGenerator.java
new file mode 100644
index 0000000..d5863d3
--- /dev/null
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDRandomIDGenerator.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.kiwi.generator;
+
+import com.fasterxml.uuid.Generators;
+import com.fasterxml.uuid.impl.RandomBasedGenerator;
+import org.apache.marmotta.kiwi.persistence.KiWiConnection;
+import org.apache.marmotta.kiwi.persistence.KiWiPersistence;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+ * Generate a long id using the least significant bits of a secure random UUDI
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class UUIDRandomIDGenerator implements IDGenerator {
+
+ RandomBasedGenerator generator;
+
+ public UUIDRandomIDGenerator() {
+ generator = Generators.randomBasedGenerator();
+ }
+
+
+ /**
+ * Initialise the generator for the given persistence and module
+ */
+ @Override
+ public void init(KiWiPersistence persistence, String scriptName) {
+ }
+
+ /**
+ * Commit the current state of memory sequences to the database using the connection passed as second argument.
+ *
+ * @param persistence
+ * @param con
+ * @throws java.sql.SQLException
+ */
+ @Override
+ public void commit(KiWiPersistence persistence, Connection con) throws SQLException {
+ }
+
+
+ /**
+ * Shut down this id generator, performing any cleanups that might be necessary.
+ *
+ * @param persistence
+ */
+ @Override
+ public void shutdown(KiWiPersistence persistence) {
+
+ }
+
+ /**
+ * Return the next unique id for the type with the given name using the generator's id generation strategy.
+ *
+ * @param name
+ * @return
+ */
+ @Override
+ public long getId(String name, KiWiConnection connection) {
+ return generator.generate().getMostSignificantBits();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDTimeIDGenerator.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDTimeIDGenerator.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDTimeIDGenerator.java
new file mode 100644
index 0000000..11662a2
--- /dev/null
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/generator/UUIDTimeIDGenerator.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.marmotta.kiwi.generator;
+
+import com.fasterxml.uuid.EthernetAddress;
+import com.fasterxml.uuid.Generators;
+import com.fasterxml.uuid.ext.FileBasedTimestampSynchronizer;
+import com.fasterxml.uuid.impl.TimeBasedGenerator;
+import org.apache.marmotta.kiwi.persistence.KiWiConnection;
+import org.apache.marmotta.kiwi.persistence.KiWiPersistence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+/**
+ * Generate a long id using the most significant bits of a time-based UUID.
+ *
+ * @author Sebastian Schaffert (sschaffert@apache.org)
+ */
+public class UUIDTimeIDGenerator implements IDGenerator {
+
+ private static Logger log = LoggerFactory.getLogger(UUIDTimeIDGenerator.class);
+
+ TimeBasedGenerator generator;
+ FileBasedTimestampSynchronizer synchronizer;
+
+ File uuid1, uuid2;
+
+ public UUIDTimeIDGenerator() {
+ try {
+ uuid1 = new File(System.getProperty("java.io.tmpdir") + File.separator + "uuid1.lck");
+ uuid2 = new File(System.getProperty("java.io.tmpdir") + File.separator + "uuid2.lck");
+
+ synchronizer = new FileBasedTimestampSynchronizer(uuid1, uuid2);
+
+ generator = Generators.timeBasedGenerator(EthernetAddress.fromInterface(), synchronizer);
+ } catch (IOException e) {
+ log.error("error initialising time-based UUID generator",e);
+ }
+ }
+
+
+ /**
+ * Initialise the generator for the given persistence and module
+ */
+ @Override
+ public void init(KiWiPersistence persistence, String scriptName) {
+ }
+
+ /**
+ * Commit the current state of memory sequences to the database using the connection passed as second argument.
+ *
+ * @param persistence
+ * @param con
+ * @throws java.sql.SQLException
+ */
+ @Override
+ public void commit(KiWiPersistence persistence, Connection con) throws SQLException {
+ }
+
+
+ /**
+ * Shut down this id generator, performing any cleanups that might be necessary.
+ *
+ * @param persistence
+ */
+ @Override
+ public void shutdown(KiWiPersistence persistence) {
+ try {
+ synchronizer.deactivate();
+
+ uuid1.delete();
+ uuid2.delete();
+ } catch (IOException e) {
+ log.error("error deactivating file synchronizer ...");
+ }
+ }
+
+ /**
+ * Return the next unique id for the type with the given name using the generator's id generation strategy.
+ *
+ * @param name
+ * @return
+ */
+ @Override
+ public synchronized long getId(String name, KiWiConnection connection) {
+ return generator.generate().getMostSignificantBits();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
index 9c7de67..4f0fd4f 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiConnection.java
@@ -17,17 +17,13 @@
*/
package org.apache.marmotta.kiwi.persistence;
+import com.google.common.base.Preconditions;
import info.aduna.iteration.*;
-
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.Element;
import org.apache.marmotta.commons.sesame.model.LiteralCommons;
import org.apache.marmotta.commons.sesame.model.Namespaces;
import org.apache.marmotta.commons.util.DateUtils;
-
-import com.google.common.base.Preconditions;
-
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.Element;
-
import org.apache.marmotta.kiwi.caching.KiWiCacheManager;
import org.apache.marmotta.kiwi.config.KiWiConfiguration;
import org.apache.marmotta.kiwi.model.caching.TripleTable;
@@ -1767,32 +1763,31 @@ public class KiWiConnection {
* @throws SQLException
*/
public long getNextSequence(String sequenceName) throws SQLException {
- if(batchCommit && persistence.getConfiguration().isMemorySequences()) {
- return persistence.incrementAndGetMemorySequence(sequenceName);
- } else {
- requireJDBCConnection();
+ return persistence.getIdGenerator().getId(sequenceName,this);
+ }
- // retrieve a new node id and set it in the node object
+ public long getDatabaseSequence(String sequenceName) throws SQLException {
+ requireJDBCConnection();
- // if there is a preparation needed to update the transaction, run it first
- if(dialect.hasStatement(sequenceName+".prep")) {
- PreparedStatement prepNodeId = getPreparedStatement(sequenceName+".prep");
- prepNodeId.executeUpdate();
- }
+ // retrieve a new node id and set it in the node object
- PreparedStatement queryNodeId = getPreparedStatement(sequenceName);
- ResultSet resultNodeId = queryNodeId.executeQuery();
- try {
- if(resultNodeId.next()) {
- return resultNodeId.getLong(1);
- } else {
- throw new SQLException("the sequence did not return a new value");
- }
- } finally {
- resultNodeId.close();
- }
+ // if there is a preparation needed to update the transaction, run it first
+ if(dialect.hasStatement(sequenceName+".prep")) {
+ PreparedStatement prepNodeId = getPreparedStatement(sequenceName+".prep");
+ prepNodeId.executeUpdate();
}
+ PreparedStatement queryNodeId = getPreparedStatement(sequenceName);
+ ResultSet resultNodeId = queryNodeId.executeQuery();
+ try {
+ if(resultNodeId.next()) {
+ return resultNodeId.getLong(1);
+ } else {
+ throw new SQLException("the sequence did not return a new value");
+ }
+ } finally {
+ resultNodeId.close();
+ }
}
@@ -1988,10 +1983,7 @@ public class KiWiConnection {
execution.execute(connection, new RetryCommand<Void>() {
@Override
public Void run() throws SQLException {
- if(persistence.getConfiguration().isCommitSequencesOnCommit() || numberOfCommits % 100 == 0) {
- commitMemorySequences();
- }
-
+ persistence.getIdGenerator().commit(persistence, getJDBCConnection());
if(tripleBatch != null && tripleBatch.size() > 0) {
flushBatch();
@@ -2010,42 +2002,6 @@ public class KiWiConnection {
}
/**
- * Store the values of all memory sequences back into the database. Should be called at least on repository shutdown
- * but possibly even when a transaction commits.
- */
- public void commitMemorySequences() throws SQLException {
- if(persistence.getMemorySequences() != null) {
- synchronized (persistence.getMemorySequences()) {
- requireJDBCConnection();
-
- Set<String> updated = persistence.getSequencesUpdated();
- persistence.setSequencesUpdated(new HashSet<String>());
-
- try {
- for(Map.Entry<String,Long> entry : persistence.getMemorySequences().asMap().entrySet()) {
- if( updated.contains(entry.getKey()) && entry.getValue() > 0) {
- PreparedStatement updateSequence = getPreparedStatement(entry.getKey()+".set");
- updateSequence.setLong(1, entry.getValue());
- if(updateSequence.execute()) {
- updateSequence.getResultSet().close();
- } else {
- updateSequence.getUpdateCount();
- }
- }
- }
- } catch(SQLException ex) {
- // MySQL deadlock state, in this case we retry anyways
- if(!"40001".equals(ex.getSQLState())) {
- log.error("SQL exception:",ex);
- }
- throw ex;
- }
- }
- }
-
- }
-
- /**
* Undoes all changes made in the current transaction
* and releases any database locks currently held
* by this <code>Connection</code> object. This method should be
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
index 98ff1b9..7ec1109 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java
@@ -17,34 +17,21 @@
*/
package org.apache.marmotta.kiwi.persistence;
-import com.google.common.util.concurrent.AtomicLongMap;
import org.apache.marmotta.kiwi.caching.KiWiCacheManager;
import org.apache.marmotta.kiwi.config.KiWiConfiguration;
-import org.apache.marmotta.kiwi.model.rdf.KiWiNode;
-import org.apache.marmotta.kiwi.model.rdf.KiWiResource;
-import org.apache.marmotta.kiwi.model.rdf.KiWiUriResource;
+import org.apache.marmotta.kiwi.generator.*;
import org.apache.marmotta.kiwi.persistence.util.ScriptRunner;
import org.apache.marmotta.kiwi.sail.KiWiValueFactory;
import org.apache.tomcat.jdbc.pool.DataSource;
import org.apache.tomcat.jdbc.pool.PoolProperties;
-import org.openrdf.model.Statement;
-import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.RepositoryResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
/**
* Add file description here!
@@ -75,20 +62,14 @@ public class KiWiPersistence {
private KiWiConfiguration configuration;
/**
- * A map holding in-memory sequences to be used for sequence caching in case the appropriate configuration option
- * is configued and batched commits are enabled.
- */
- private AtomicLongMap<String> memorySequences;
-
- private ReentrantLock sequencesLock;
-
-
- /**
* A reference to the value factory used to access this store. Used for notifications when to flush batches.
*/
private KiWiValueFactory valueFactory;
+ private IDGenerator idGenerator;
+
+
/**
* This lock allows setting the backend into maintenance mode (by locking the write lock), which essentially
* grants an exclusive access to the database. This is currently used by the garbage collector, but can also
@@ -100,9 +81,6 @@ public class KiWiPersistence {
private boolean initialized = false;
- // keep track which memory sequences have been updated and need to be written back
- private Set<String> sequencesUpdated;
-
@Deprecated
public KiWiPersistence(String name, String jdbcUrl, String db_user, String db_password, KiWiDialect dialect) {
this(new KiWiConfiguration(name,jdbcUrl,db_user,db_password,dialect));
@@ -111,8 +89,6 @@ public class KiWiPersistence {
public KiWiPersistence(KiWiConfiguration configuration) {
this.configuration = configuration;
this.maintenance = false;
- this.sequencesLock = new ReentrantLock();
- this.sequencesUpdated = new HashSet<>();
}
public void initialise() {
@@ -207,50 +183,26 @@ public class KiWiPersistence {
* Initialise in-memory sequences if the feature is enabled.
*/
public void initSequences(String scriptName) {
- if(configuration.isBatchCommit() && configuration.isMemorySequences()) {
- sequencesLock.lock();
- try {
- if(memorySequences == null) {
- memorySequences = AtomicLongMap.create();
- }
-
- try {
- Connection con = getJDBCConnection(true);
- try {
- for(String sequenceName : getDialect().listSequences(scriptName)) {
-
- // load sequence value from database
- // if there is a preparation needed to update the transaction, run it first
- if(getDialect().hasStatement(sequenceName+".prep")) {
- PreparedStatement prepNodeId = con.prepareStatement(getDialect().getStatement(sequenceName+".prep"));
- prepNodeId.executeUpdate();
- prepNodeId.close();
- }
-
- PreparedStatement queryNodeId = con.prepareStatement(getDialect().getStatement(sequenceName));
- ResultSet resultNodeId = queryNodeId.executeQuery();
- try {
- if(resultNodeId.next()) {
- memorySequences.put(sequenceName,resultNodeId.getLong(1)-1);
- } else {
- throw new SQLException("the sequence did not return a new value");
- }
- } finally {
- resultNodeId.close();
- }
-
- con.commit();
- }
- } finally {
- releaseJDBCConnection(con);
- }
- } catch(SQLException ex) {
- log.warn("database error: could not initialise in-memory sequences",ex);
- }
- } finally {
- sequencesLock.unlock();
- }
+ switch (configuration.getIdGeneratorType()) {
+ case DATABASE_SEQUENCE:
+ idGenerator = new DatabaseSequenceIDGenerator();
+ break;
+ case MEMORY_SEQUENCE:
+ idGenerator = new MemorySequenceIDGenerator();
+ break;
+ case UUID_TIME:
+ idGenerator = new UUIDTimeIDGenerator();
+ break;
+ case UUID_RANDOM:
+ idGenerator = new UUIDRandomIDGenerator();
+ break;
+ case SNOWFLAKE:
+ idGenerator = new SnowflakeIDGenerator();
+ break;
+ default:
+ idGenerator = new DatabaseSequenceIDGenerator();
}
+ idGenerator.init(this,scriptName);
}
public void logPoolInfo() throws SQLException {
@@ -510,29 +462,12 @@ public class KiWiPersistence {
public void shutdown() {
initialized = false;
- if(!droppedDatabase && !configuration.isCommitSequencesOnCommit()) {
- log.info("storing in-memory sequences in database ...");
- try {
- KiWiConnection connection = getConnection();
- try {
- connection.commitMemorySequences();
- connection.commit();
- } finally {
- connection.close();
- }
- } catch (SQLException e) {
- log.error("could not store back values of in-memory sequences", e);
- }
- }
-
-
+ idGenerator.shutdown(this);
garbageCollector.shutdown();
cacheManager.shutdown();
connectionPool.close();
connectionPool = null;
- memorySequences = null;
-
}
/**
@@ -555,20 +490,6 @@ public class KiWiPersistence {
return configuration;
}
- public AtomicLongMap<String> getMemorySequences() {
- return memorySequences;
- }
-
- public long incrementAndGetMemorySequence(String name) {
- sequencesUpdated.add(name);
-
- if(memorySequences != null) {
- return memorySequences.incrementAndGet(name);
- } else {
- return 0;
- }
- }
-
public void garbageCollect() throws SQLException {
this.garbageCollector.garbageCollect();
@@ -578,11 +499,7 @@ public class KiWiPersistence {
return garbageCollector.checkConsistency();
}
- public Set<String> getSequencesUpdated() {
- return sequencesUpdated;
- }
-
- public void setSequencesUpdated(Set<String> sequencesUpdated) {
- this.sequencesUpdated = sequencesUpdated;
+ public IDGenerator getIdGenerator() {
+ return idGenerator;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 2bfcc52..61c02e0 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -976,6 +976,19 @@
<version>7.0.42</version>
</dependency>
+ <!--Secure Unique IDs -->
+ <dependency>
+ <groupId>com.fasterxml.uuid</groupId>
+ <artifactId>java-uuid-generator</artifactId>
+ <version>3.1.3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<!-- Test Support -->
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
----------------------------------------------------------------------
diff --git a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java b/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
index 29061c3..2684984 100644
--- a/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
+++ b/platform/marmotta-core/src/main/java/org/apache/marmotta/platform/core/services/triplestore/SesameServiceImpl.java
@@ -19,14 +19,7 @@ package org.apache.marmotta.platform.core.services.triplestore;
import edu.emory.mathcs.backport.java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.marmotta.kiwi.config.KiWiConfiguration;
-import org.apache.marmotta.platform.core.api.config.ConfigurationService;
-import org.apache.marmotta.platform.core.api.triplestore.NotifyingSailProvider;
-import org.apache.marmotta.platform.core.api.triplestore.SesameService;
-import org.apache.marmotta.platform.core.api.triplestore.StandardSailProvider;
-import org.apache.marmotta.platform.core.api.triplestore.TransactionalSailProvider;
-import org.apache.marmotta.platform.core.qualifiers.event.transaction.AfterCommit;
-import org.apache.marmotta.platform.core.qualifiers.event.transaction.AfterRollback;
-import org.apache.marmotta.platform.core.qualifiers.event.transaction.BeforeCommit;
+import org.apache.marmotta.kiwi.generator.IDGeneratorType;
import org.apache.marmotta.kiwi.persistence.KiWiDialect;
import org.apache.marmotta.kiwi.persistence.h2.H2Dialect;
import org.apache.marmotta.kiwi.persistence.mysql.MySQLDialect;
@@ -36,6 +29,14 @@ import org.apache.marmotta.kiwi.transactions.api.TransactionListener;
import org.apache.marmotta.kiwi.transactions.api.TransactionalSail;
import org.apache.marmotta.kiwi.transactions.model.TransactionData;
import org.apache.marmotta.kiwi.transactions.sail.KiWiTransactionalSail;
+import org.apache.marmotta.platform.core.api.config.ConfigurationService;
+import org.apache.marmotta.platform.core.api.triplestore.NotifyingSailProvider;
+import org.apache.marmotta.platform.core.api.triplestore.SesameService;
+import org.apache.marmotta.platform.core.api.triplestore.StandardSailProvider;
+import org.apache.marmotta.platform.core.api.triplestore.TransactionalSailProvider;
+import org.apache.marmotta.platform.core.qualifiers.event.transaction.AfterCommit;
+import org.apache.marmotta.platform.core.qualifiers.event.transaction.AfterRollback;
+import org.apache.marmotta.platform.core.qualifiers.event.transaction.BeforeCommit;
import org.openrdf.model.ValueFactory;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;
@@ -155,7 +156,17 @@ public class SesameServiceImpl implements SesameService {
configuration.setQueryLoggingEnabled(configurationService.getBooleanConfiguration("database.debug.slowqueries",false));
configuration.setBatchCommit(batchCommit);
configuration.setBatchSize(configurationService.getIntConfiguration("database.batchsize",10000));
- configuration.setMemorySequences(configurationService.getBooleanConfiguration("database.memsequences",true));
+
+ String generatorType = configurationService.getStringConfiguration("database.generator", "uuid-time");
+ if("uuid-time".equals(generatorType)) {
+ configuration.setIdGeneratorType(IDGeneratorType.UUID_TIME);
+ } else if("uuid-random".equals(generatorType)) {
+ configuration.setIdGeneratorType(IDGeneratorType.UUID_RANDOM);
+ } else if("sequence".equals(generatorType)) {
+ configuration.setIdGeneratorType(IDGeneratorType.DATABASE_SEQUENCE);
+ } else if("memory".equals(generatorType)) {
+ configuration.setIdGeneratorType(IDGeneratorType.MEMORY_SEQUENCE);
+ }
store = new KiWiStore(configuration);
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/platform/marmotta-core/src/main/resources/config-defaults.properties
----------------------------------------------------------------------
diff --git a/platform/marmotta-core/src/main/resources/config-defaults.properties b/platform/marmotta-core/src/main/resources/config-defaults.properties
index 16caabb..8b1c0d9 100644
--- a/platform/marmotta-core/src/main/resources/config-defaults.properties
+++ b/platform/marmotta-core/src/main/resources/config-defaults.properties
@@ -169,9 +169,8 @@ database.batchcommit = true
# the maximum size of a batch before it is committed to the database; only applicable if batchcommit is enabled
database.batchsize = 10000
-# turn on in-memory sequences (EXPERIMENTAL); if enabled, sequences are pre-loaded once and then updated in memory
-# and only written back if a transaction commits
-database.memsequences = true
+# generator to use for creating database IDs; possible values are uuid-time, uuid-random, sequence, memory, snowflake
+database.generator = snowflake
# Hibernate-specific configuration for H2
database.h2.driver = org.h2.Driver
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4b8b8680/platform/marmotta-core/src/main/resources/config-descriptions.properties
----------------------------------------------------------------------
diff --git a/platform/marmotta-core/src/main/resources/config-descriptions.properties b/platform/marmotta-core/src/main/resources/config-descriptions.properties
index f0c80c7..4f58780 100644
--- a/platform/marmotta-core/src/main/resources/config-descriptions.properties
+++ b/platform/marmotta-core/src/main/resources/config-descriptions.properties
@@ -173,10 +173,11 @@ database.batchsize.description = the maximum size of a batch before it is commit
batchcommit is enabled
database.batchsize.type = java.lang.Integer(10|0|*)
-database.memsequences.description = turn on in-memory sequences (EXPERIMENTAL); if enabled, sequences are pre-loaded \
- once and then updated in memory and only written back if a transaction commits
-database.memsequences.type = java.lang.Boolean
-
+database.generator.description = generator for creating database identifiers; uuid-time creates unique identifiers based \
+ on time and machine, uuid-random creates random identifiers, sequence uses a database sequence, memory loads a database \
+ sequence and increments in-memory (fast but requires exclusive database access), snowflake uses the Twitter Snowflake \
+ algorithm (fast and reliable)
+database.generator.type = java.lang.Enum("uuid-time"|"uuid-random"|"sequence"|"memory"|"snowflake")
###############################################################################
# LMF importer configuration