You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ignite.apache.org by GitBox <gi...@apache.org> on 2022/08/18 12:02:58 UTC
[GitHub] [ignite-3] rpuch opened a new pull request, #1022: IGNITE-17336 Spill-out to disk support for volatile RAFT log storage
rpuch opened a new pull request, #1022:
URL: https://github.com/apache/ignite-3/pull/1022
https://issues.apache.org/jira/browse/IGNITE-17336
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [ignite-3] SammyVimes commented on a diff in pull request #1022: IGNITE-17336 Spill-out to disk support for volatile RAFT log storage
Posted by GitBox <gi...@apache.org>.
SammyVimes commented on code in PR #1022:
URL: https://github.com/apache/ignite-3/pull/1022#discussion_r950056278
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/impl/VolatileLogStorageFactoryCreator.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.storage.impl;
+
+import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.ignite.configuration.schemas.table.LogStorageBudgetView;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.manager.IgniteComponent;
+import org.apache.ignite.internal.raft.storage.LogStorageFactory;
+import org.apache.ignite.internal.thread.NamedThreadFactory;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.lang.IgniteInternalException;
+import org.apache.ignite.raft.jraft.util.ExecutorServiceHelper;
+import org.apache.ignite.raft.jraft.util.Platform;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.CompressionType;
+import org.rocksdb.DBOptions;
+import org.rocksdb.Env;
+import org.rocksdb.Options;
+import org.rocksdb.Priority;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.util.SizeUnit;
+
+/**
+ * {@link LogStorageFactoryCreator} for volatile log storage.
+ */
+public class VolatileLogStorageFactoryCreator implements LogStorageFactoryCreator, IgniteComponent {
+ private static final IgniteLogger LOG = Loggers.forClass(VolatileLogStorageFactoryCreator.class);
+
+ /** Database path. */
+ private final Path spillOutPath;
+
+ /** Database options. */
+ private DBOptions dbOptions;
+
+ /** Shared db instance. */
+ private RocksDB db;
+
+ /** Shared data column family handle. */
+ private ColumnFamilyHandle columnFamily;
+
+ /** Executor for spill-out RocksDB tasks. */
+ private final ExecutorService executorService;
+
+ /**
+ * Create a new instance.
+ *
+ * @param spillOutPath Path at which to put spill-out data.
+ */
+ public VolatileLogStorageFactoryCreator(Path spillOutPath) {
+ this.spillOutPath = Objects.requireNonNull(spillOutPath);
+
+ executorService = Executors.newFixedThreadPool(
+ Runtime.getRuntime().availableProcessors() * 2,
+ new NamedThreadFactory("raft-volatile-log-rocksdb-spillout-pool", LOG)
+ );
+ }
+
+ @Override
+ public void start() {
+ try {
+ Files.createDirectories(spillOutPath);
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to create directory: " + this.spillOutPath, e);
+ }
+
+ wipeOutDb();
+
+ dbOptions = createDbOptions();
+ ColumnFamilyOptions cfOption = createColumnFamilyOptions();
+
+ List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
+
+ List<ColumnFamilyDescriptor> columnFamilyDescriptors = List.of(
+ new ColumnFamilyDescriptor(DEFAULT_COLUMN_FAMILY, cfOption)
+ );
+
+ try {
+ db = RocksDB.open(this.dbOptions, this.spillOutPath.toString(), columnFamilyDescriptors, columnFamilyHandles);
+
+ // Setup rocks thread pools to utilize all the available cores as the database is shared among
+ // all the raft groups
+ Env env = db.getEnv();
+ // Setup background flushes pool
+ env.setBackgroundThreads(Runtime.getRuntime().availableProcessors(), Priority.HIGH);
+ // Setup background compactions pool
+ env.setBackgroundThreads(Runtime.getRuntime().availableProcessors(), Priority.LOW);
+
+ assert (columnFamilyHandles.size() == 1);
+ this.columnFamily = columnFamilyHandles.get(0);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void wipeOutDb() {
+ try (var options = new Options()) {
+ RocksDB.destroyDB(spillOutPath.toString(), options);
+ } catch (RocksDBException e) {
+ throw new IgniteInternalException("Cannot destroy spill-out RocksDB at " + spillOutPath, e);
+ }
+ }
+
+ /**
+ * Creates database options.
+ *
+ * @return Default database options.
+ */
+ private static DBOptions createDbOptions() {
+ return new DBOptions()
+ .setMaxBackgroundJobs(Runtime.getRuntime().availableProcessors() * 2)
+ .setCreateIfMissing(true)
+ .setCreateMissingColumnFamilies(true);
+ }
+
+ /**
+ * Creates column family options.
+ *
+ * @return Default column family options.
+ */
+ private static ColumnFamilyOptions createColumnFamilyOptions() {
+ var opts = new ColumnFamilyOptions();
+
+ opts.setWriteBufferSize(64 * SizeUnit.MB);
+ opts.setMaxWriteBufferNumber(5);
+ opts.setMinWriteBufferNumberToMerge(1);
+ opts.setLevel0FileNumCompactionTrigger(50);
+ opts.setLevel0SlowdownWritesTrigger(100);
+ opts.setLevel0StopWritesTrigger(200);
+ // Size of level 0 which is (in stable state) equal to
+ // WriteBufferSize * MinWriteBufferNumberToMerge * Level0FileNumCompactionTrigger
+ opts.setMaxBytesForLevelBase(3200 * SizeUnit.MB);
Review Comment:
I think this place is worth adding a TODO with a message like "add spill-out parameters"
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/impl/LocalLogStorageFactory.java:
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.storage.impl;
+
+import org.apache.ignite.internal.raft.storage.LogStorageFactory;
+import org.apache.ignite.raft.jraft.option.RaftOptions;
+import org.apache.ignite.raft.jraft.storage.LogStorage;
+import org.apache.ignite.raft.jraft.storage.impl.LocalLogStorage;
+
+/**
+ * LogStorageFactory that always creates instances of {@link LocalLogStorage}.
Review Comment:
```suggestion
* LogStorageFactory that creates instances of {@link LocalLogStorage}.
```
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/storage/impl/VolatileLogStorageFactoryCreator.java:
##########
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.storage.impl;
+
+import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.ignite.configuration.schemas.table.LogStorageBudgetView;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.manager.IgniteComponent;
+import org.apache.ignite.internal.raft.storage.LogStorageFactory;
+import org.apache.ignite.internal.thread.NamedThreadFactory;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.lang.IgniteInternalException;
+import org.apache.ignite.raft.jraft.util.ExecutorServiceHelper;
+import org.apache.ignite.raft.jraft.util.Platform;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.CompressionType;
+import org.rocksdb.DBOptions;
+import org.rocksdb.Env;
+import org.rocksdb.Options;
+import org.rocksdb.Priority;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.util.SizeUnit;
+
+/**
+ * {@link LogStorageFactoryCreator} for volatile log storage.
+ */
+public class VolatileLogStorageFactoryCreator implements LogStorageFactoryCreator, IgniteComponent {
+ private static final IgniteLogger LOG = Loggers.forClass(VolatileLogStorageFactoryCreator.class);
+
+ /** Database path. */
+ private final Path spillOutPath;
+
+ /** Database options. */
+ private DBOptions dbOptions;
+
+ /** Shared db instance. */
+ private RocksDB db;
+
+ /** Shared data column family handle. */
+ private ColumnFamilyHandle columnFamily;
+
+ /** Executor for spill-out RocksDB tasks. */
+ private final ExecutorService executorService;
+
+ /**
+ * Create a new instance.
+ *
+ * @param spillOutPath Path at which to put spill-out data.
+ */
+ public VolatileLogStorageFactoryCreator(Path spillOutPath) {
+ this.spillOutPath = Objects.requireNonNull(spillOutPath);
+
+ executorService = Executors.newFixedThreadPool(
+ Runtime.getRuntime().availableProcessors() * 2,
+ new NamedThreadFactory("raft-volatile-log-rocksdb-spillout-pool", LOG)
+ );
+ }
+
+ @Override
+ public void start() {
+ try {
+ Files.createDirectories(spillOutPath);
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to create directory: " + this.spillOutPath, e);
+ }
+
+ wipeOutDb();
+
+ dbOptions = createDbOptions();
+ ColumnFamilyOptions cfOption = createColumnFamilyOptions();
+
+ List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
+
+ List<ColumnFamilyDescriptor> columnFamilyDescriptors = List.of(
+ new ColumnFamilyDescriptor(DEFAULT_COLUMN_FAMILY, cfOption)
+ );
+
+ try {
+ db = RocksDB.open(this.dbOptions, this.spillOutPath.toString(), columnFamilyDescriptors, columnFamilyHandles);
+
+ // Setup rocks thread pools to utilize all the available cores as the database is shared among
+ // all the raft groups
+ Env env = db.getEnv();
+ // Setup background flushes pool
+ env.setBackgroundThreads(Runtime.getRuntime().availableProcessors(), Priority.HIGH);
+ // Setup background compactions pool
Review Comment:
```suggestion
// Setup background compactions pool
```
##########
modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/impl/Logs.java:
##########
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.raft.jraft.storage.impl;
+
+import java.util.List;
+import org.apache.ignite.raft.jraft.Lifecycle;
+import org.apache.ignite.raft.jraft.entity.LogEntry;
+import org.apache.ignite.raft.jraft.option.LogStorageOptions;
+import org.apache.ignite.raft.jraft.storage.Storage;
+
+/**
+ * Log entry storage (used for internal needs of {@link VolatileLogStorage}).
+ */
+interface Logs extends Lifecycle<LogStorageOptions>, Storage {
+ /**
+ * Get logEntry by index.
+ */
+ LogEntry getEntry(final long index);
+
+ /**
+ * Append entries to log.
+ */
+ void appendEntry(final LogEntry entry);
+
+ /**
+ * Append entries to log, return append success number.
+ */
+ void appendEntries(final List<LogEntry> entries);
+
+ /**
+ * Delete logs from storage's head, [first_log_index, first_index_kept) will be discarded.
+ */
+ void truncatePrefix(final long firstIndexKept);
+
+ /**
+ * Delete uncommitted logs from storage's tail, (last_index_kept, last_log_index] will be discarded.
+ */
+ void truncateSuffix(final long lastIndexKept);
+
+ /**
+ * Drop all the existing logs and reset next log index to |next_log_index|. This function is called after installing
Review Comment:
I'm curious about `next_log_index`, `first_log_index`, `last_index_kept` and so on. Are these variables defined in raft paper?
##########
modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/impl/Logs.java:
##########
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.raft.jraft.storage.impl;
+
+import java.util.List;
+import org.apache.ignite.raft.jraft.Lifecycle;
+import org.apache.ignite.raft.jraft.entity.LogEntry;
+import org.apache.ignite.raft.jraft.option.LogStorageOptions;
+import org.apache.ignite.raft.jraft.storage.Storage;
+
+/**
+ * Log entry storage (used for internal needs of {@link VolatileLogStorage}).
+ */
+interface Logs extends Lifecycle<LogStorageOptions>, Storage {
+ /**
+ * Get logEntry by index.
+ */
+ LogEntry getEntry(final long index);
+
+ /**
+ * Append entries to log.
+ */
+ void appendEntry(final LogEntry entry);
+
+ /**
+ * Append entries to log, return append success number.
Review Comment:
Probably doesn't return anything
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [ignite-3] rpuch commented on a diff in pull request #1022: IGNITE-17336 Spill-out to disk support for volatile RAFT log storage
Posted by GitBox <gi...@apache.org>.
rpuch commented on code in PR #1022:
URL: https://github.com/apache/ignite-3/pull/1022#discussion_r951130686
##########
modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/impl/Logs.java:
##########
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.raft.jraft.storage.impl;
+
+import java.util.List;
+import org.apache.ignite.raft.jraft.Lifecycle;
+import org.apache.ignite.raft.jraft.entity.LogEntry;
+import org.apache.ignite.raft.jraft.option.LogStorageOptions;
+import org.apache.ignite.raft.jraft.storage.Storage;
+
+/**
+ * Log entry storage (used for internal needs of {@link VolatileLogStorage}).
+ */
+interface Logs extends Lifecycle<LogStorageOptions>, Storage {
+ /**
+ * Get logEntry by index.
+ */
+ LogEntry getEntry(final long index);
+
+ /**
+ * Append entries to log.
+ */
+ void appendEntry(final LogEntry entry);
+
+ /**
+ * Append entries to log, return append success number.
+ */
+ void appendEntries(final List<LogEntry> entries);
+
+ /**
+ * Delete logs from storage's head, [first_log_index, first_index_kept) will be discarded.
+ */
+ void truncatePrefix(final long firstIndexKept);
+
+ /**
+ * Delete uncommitted logs from storage's tail, (last_index_kept, last_log_index] will be discarded.
+ */
+ void truncateSuffix(final long lastIndexKept);
+
+ /**
+ * Drop all the existing logs and reset next log index to |next_log_index|. This function is called after installing
Review Comment:
Those are internal details of JRaft implementation, not terms from the raft paper
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [ignite-3] SammyVimes commented on a diff in pull request #1022: IGNITE-17336 Spill-out to disk support for volatile RAFT log storage
Posted by GitBox <gi...@apache.org>.
SammyVimes commented on code in PR #1022:
URL: https://github.com/apache/ignite-3/pull/1022#discussion_r951233704
##########
modules/raft/src/main/java/org/apache/ignite/raft/jraft/storage/impl/Logs.java:
##########
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.raft.jraft.storage.impl;
+
+import java.util.List;
+import org.apache.ignite.raft.jraft.Lifecycle;
+import org.apache.ignite.raft.jraft.entity.LogEntry;
+import org.apache.ignite.raft.jraft.option.LogStorageOptions;
+import org.apache.ignite.raft.jraft.storage.Storage;
+
+/**
+ * Log entry storage (used for internal needs of {@link VolatileLogStorage}).
+ */
+interface Logs extends Lifecycle<LogStorageOptions>, Storage {
+ /**
+ * Get logEntry by index.
+ */
+ LogEntry getEntry(final long index);
+
+ /**
+ * Append entries to log.
+ */
+ void appendEntry(final LogEntry entry);
+
+ /**
+ * Append entries to log, return append success number.
+ */
+ void appendEntries(final List<LogEntry> entries);
+
+ /**
+ * Delete logs from storage's head, [first_log_index, first_index_kept) will be discarded.
+ */
+ void truncatePrefix(final long firstIndexKept);
+
+ /**
+ * Delete uncommitted logs from storage's tail, (last_index_kept, last_log_index] will be discarded.
+ */
+ void truncateSuffix(final long lastIndexKept);
+
+ /**
+ * Drop all the existing logs and reset next log index to |next_log_index|. This function is called after installing
Review Comment:
Ok!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [ignite-3] SammyVimes merged pull request #1022: IGNITE-17336 Spill-out to disk support for volatile RAFT log storage
Posted by GitBox <gi...@apache.org>.
SammyVimes merged PR #1022:
URL: https://github.com/apache/ignite-3/pull/1022
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org