You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/08/10 10:31:49 UTC
ignite git commit: IGNITE-5961 - Align pages in LFS partition files
to pageSize
Repository: ignite
Updated Branches:
refs/heads/master b5ad72969 -> 3919d8035
IGNITE-5961 - Align pages in LFS partition files to pageSize
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3919d803
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3919d803
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3919d803
Branch: refs/heads/master
Commit: 3919d8035fb172e1546f7a3ecea0f29f8e1e2214
Parents: b5ad729
Author: Ivan Rakov <iv...@gmail.com>
Authored: Thu Aug 10 13:31:23 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Aug 10 13:31:23 2017 +0300
----------------------------------------------------------------------
.../internal/pagemem/store/PageStore.java | 5 +
.../cache/persistence/file/FilePageStore.java | 54 ++++++---
.../persistence/file/FilePageStoreFactory.java | 35 ++++++
.../persistence/file/FilePageStoreManager.java | 17 +--
.../cache/persistence/file/FilePageStoreV2.java | 53 +++++++++
.../file/FileVersionCheckingFactory.java | 116 +++++++++++++++++++
...gnitePdsRecoveryAfterFileCorruptionTest.java | 2 +-
7 files changed, 251 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
index 4698a6b..f6e577c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
@@ -95,4 +95,9 @@ public interface PageStore {
* @throws IgniteCheckedException If sync failed (IO error occurred).
*/
public void ensure() throws IgniteCheckedException;
+
+ /**
+ * @return Page store version.
+ */
+ public int version();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index a7ca13c..7fe1ffe 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -45,10 +45,10 @@ public class FilePageStore implements PageStore {
private static final long SIGNATURE = 0xF19AC4FE60C530B8L;
/** File version. */
- private static final int VERSION = 1;
+ public static final int VERSION = 1;
/** Allocated field offset. */
- public static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/;
+ static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/;
/** */
private final File cfgFile;
@@ -57,7 +57,7 @@ public class FilePageStore implements PageStore {
private final byte type;
/** Database configuration. */
- private final MemoryConfiguration dbCfg;
+ protected final MemoryConfiguration dbCfg;
/** Factory to provide I/O interfaces for read/write operations with files */
private final FileIOFactory ioFactory;
@@ -103,20 +103,36 @@ public class FilePageStore implements PageStore {
/** {@inheritDoc} */
@Override public boolean exists() {
- return cfgFile.exists() && cfgFile.length() > HEADER_SIZE;
+ return cfgFile.exists() && cfgFile.length() > headerSize();
}
/**
+ * Size of page store header.
+ */
+ public int headerSize() {
+ return HEADER_SIZE;
+ }
+
+ /**
+ * Page store version.
+ */
+ public int version() {
+ return VERSION;
+ }
+
+ /**
+ * Creates header for current version file store. Doesn't init the store.
+ *
* @param type Type.
* @param pageSize Page size.
* @return Byte buffer instance.
*/
- public static ByteBuffer header(byte type, int pageSize) {
- ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+ public ByteBuffer header(byte type, int pageSize) {
+ ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
hdr.putLong(SIGNATURE);
- hdr.putInt(VERSION);
+ hdr.putInt(version());
hdr.put(type);
@@ -142,7 +158,7 @@ public class FilePageStore implements PageStore {
}
//there is 'super' page in every file
- return HEADER_SIZE + dbCfg.getPageSize();
+ return headerSize() + dbCfg.getPageSize();
}
/**
@@ -150,7 +166,7 @@ public class FilePageStore implements PageStore {
*/
private long checkFile() throws IgniteCheckedException {
try {
- ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
while (hdr.remaining() > 0)
fileIO.read(hdr);
@@ -166,9 +182,9 @@ public class FilePageStore implements PageStore {
int ver = hdr.getInt();
- if (VERSION != ver)
+ if (version() != ver)
throw new IgniteCheckedException("Failed to verify store file (invalid file version)" +
- " [expectedVersion=" + VERSION +
+ " [expectedVersion=" + version() +
", fileVersion=" + ver + "]");
byte type = hdr.get();
@@ -187,10 +203,10 @@ public class FilePageStore implements PageStore {
long fileSize = cfgFile.length();
- if (fileSize == HEADER_SIZE) // Every file has a special meta page.
- fileSize = pageSize + HEADER_SIZE;
+ if (fileSize == headerSize()) // Every file has a special meta page.
+ fileSize = pageSize + headerSize();
- if ((fileSize - HEADER_SIZE) % pageSize != 0)
+ if ((fileSize - headerSize()) % pageSize != 0)
throw new IgniteCheckedException("Failed to verify store file (invalid file size)" +
" [fileSize=" + U.hexLong(fileSize) +
", pageSize=" + U.hexLong(pageSize) + ']');
@@ -346,9 +362,9 @@ public class FilePageStore implements PageStore {
init();
try {
- assert buf.remaining() == HEADER_SIZE;
+ assert buf.remaining() == headerSize();
- int len = HEADER_SIZE;
+ int len = headerSize();
long off = 0;
@@ -425,7 +441,7 @@ public class FilePageStore implements PageStore {
long off = pageOffset(pageId);
- assert (off >= 0 && off + pageSize <= allocated.get() + HEADER_SIZE) || recover :
+ assert (off >= 0 && off + pageSize <= allocated.get() + headerSize()) || recover :
"off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId);
assert pageBuf.capacity() == pageSize;
@@ -463,7 +479,7 @@ public class FilePageStore implements PageStore {
/** {@inheritDoc} */
@Override public long pageOffset(long pageId) {
- return (long) PageIdUtils.pageIndex(pageId) * pageSize + HEADER_SIZE;
+ return (long) PageIdUtils.pageIndex(pageId) * pageSize + headerSize();
}
/** {@inheritDoc} */
@@ -494,7 +510,7 @@ public class FilePageStore implements PageStore {
long off = allocPage();
- return off / pageSize;
+ return (off - headerSize()) / pageSize;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
new file mode 100644
index 0000000..d97ab26
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
@@ -0,0 +1,35 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+
+/**
+ *
+ */
+public interface FilePageStoreFactory {
+ /**
+ * Creates instance of FilePageStore based on given file.
+ *
+ * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA}.
+ * @param file File Page store file.
+ */
+ public FilePageStore createPageStore(byte type, File file) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index e2ad070..0041ea6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -365,21 +365,16 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
if (dirExisted && !idxFile.exists())
grpsWithoutIdx.add(grpDesc.groupId());
- FilePageStore idxStore = new FilePageStore(
- PageMemory.FLAG_IDX,
- idxFile,
- pstCfg.getFileIOFactory(),
- cctx.kernalContext().config().getMemoryConfiguration());
+ FileVersionCheckingFactory pageStoreFactory = new FileVersionCheckingFactory(
+ pstCfg.getFileIOFactory(), igniteCfg.getMemoryConfiguration());
+
+ FilePageStore idxStore = pageStoreFactory.createPageStore(PageMemory.FLAG_IDX, idxFile);
FilePageStore[] partStores = new FilePageStore[grpDesc.config().getAffinity().partitions()];
for (int partId = 0; partId < partStores.length; partId++) {
- FilePageStore partStore = new FilePageStore(
- PageMemory.FLAG_DATA,
- new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)),
- pstCfg.getFileIOFactory(),
- cctx.kernalContext().config().getMemoryConfiguration()
- );
+ FilePageStore partStore = pageStoreFactory.createPageStore(
+ PageMemory.FLAG_DATA, new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)));
partStores[partId] = partStore;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
new file mode 100644
index 0000000..5d044ec
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
@@ -0,0 +1,53 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ *
+ */
+public class FilePageStoreV2 extends FilePageStore {
+ /** File version. */
+ public static final int VERSION = 2;
+
+ /** Header size. */
+ private final int hdrSize;
+
+ /**
+ * @param type Type.
+ * @param file File.
+ * @param factory Factory.
+ * @param cfg Config.
+ */
+ public FilePageStoreV2(byte type, File file, FileIOFactory factory, MemoryConfiguration cfg) {
+ super(type, file, factory, cfg);
+
+ hdrSize = cfg.getPageSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int headerSize() {
+ return hdrSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int version() {
+ return VERSION;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
new file mode 100644
index 0000000..53bd802
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
@@ -0,0 +1,116 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ * Checks version in files if it's present on the disk, creates store with latest version otherwise.
+ */
+public class FileVersionCheckingFactory implements FilePageStoreFactory {
+ /** Property to override latest version. Should be used only in tests. */
+ public static final String LATEST_VERSION_OVERRIDE_PROPERTY = "file.page.store.latest.version.override";
+
+ /** Latest page store version. */
+ public final static int LATEST_VERSION = 2;
+
+ /** Factory to provide I/O interfaces for read/write operations with files. */
+ private final FileIOFactory fileIOFactory;
+
+ /** Memory configuration. */
+ private final MemoryConfiguration memCfg;
+
+ /**
+ * @param fileIOFactory File io factory.
+ * @param memCfg Memory configuration.
+ */
+ public FileVersionCheckingFactory(
+ FileIOFactory fileIOFactory, MemoryConfiguration memCfg) {
+ this.fileIOFactory = fileIOFactory;
+ this.memCfg = memCfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FilePageStore createPageStore(byte type, File file) throws IgniteCheckedException {
+ if (!file.exists())
+ return createPageStore(type, file, latestVersion());
+
+ try (FileIO fileIO = fileIOFactory.create(file, "r")) {
+ int minHdr = FilePageStore.HEADER_SIZE;
+
+ if (fileIO.size() < minHdr)
+ return createPageStore(type, file, latestVersion());
+
+ ByteBuffer hdr = ByteBuffer.allocate(minHdr).order(ByteOrder.LITTLE_ENDIAN);
+
+ while (hdr.remaining() > 0)
+ fileIO.read(hdr);
+
+ hdr.rewind();
+
+ hdr.getLong(); // Read signature
+
+ int ver = hdr.getInt();
+
+ return createPageStore(type, file, ver);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Error while creating file page store [file=" + file + "]:", e);
+ }
+ }
+
+ /**
+ * Resolves latest page store version.
+ */
+ public int latestVersion() {
+ int latestVer = LATEST_VERSION;
+
+ try {
+ latestVer = Integer.parseInt(System.getProperty(LATEST_VERSION_OVERRIDE_PROPERTY));
+ } catch (NumberFormatException e) {
+ // No override.
+ }
+
+ return latestVer;
+ }
+
+ /**
+ * Instantiates specific version of FilePageStore.
+ *
+ * @param type Type.
+ * @param file File.
+ * @param ver Version.
+ */
+ public FilePageStore createPageStore(byte type, File file, int ver) throws IgniteCheckedException {
+ switch (ver) {
+ case FilePageStore.VERSION:
+ return new FilePageStore(type, file, fileIOFactory, memCfg);
+
+ case FilePageStoreV2.VERSION:
+ return new FilePageStoreV2(type, file, fileIOFactory, memCfg);
+
+ default:
+ throw new IllegalArgumentException("Unknown version of file page store: " + ver);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
index c248c35..11d5eef 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
@@ -194,7 +194,7 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
long size = fileIO.size();
- fileIO.write(ByteBuffer.allocate((int)size - FilePageStore.HEADER_SIZE), FilePageStore.HEADER_SIZE);
+ fileIO.write(ByteBuffer.allocate((int)size - filePageStore.headerSize()), filePageStore.headerSize());
fileIO.force();
}