You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2016/04/28 10:52:31 UTC

[01/50] ignite git commit: ignite-db - minor refactor

Repository: ignite
Updated Branches:
  refs/heads/ignite-db-x-10884 ce4698f2d -> fc7e0630b


ignite-db - minor refactor


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/14ffbf0b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/14ffbf0b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/14ffbf0b

Branch: refs/heads/ignite-db-x-10884
Commit: 14ffbf0bcc48ffa4a652e523da74e29d53c7ead3
Parents: 213fc70
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 20 21:10:41 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 20 21:10:41 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 72 ++++++++++----------
 1 file changed, 36 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/14ffbf0b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 864dae8..37d6d06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -621,41 +621,12 @@ public abstract class BPlusTree<L, T extends L> {
     }
 
     /**
-     * Initialize the given operation.
-     *
-     * !!! Symmetrically with this method must be called {@link Get#releaseMeta()} in {@code finally} block.
-     *
-     * @param g Operation.
-     */
-    private void initOperation(Get g) throws IgniteCheckedException {
-        if (g.meta == null)
-            g.meta = page(metaPageId);
-
-        int rootLvl;
-        long rootId;
-
-        ByteBuffer buf = g.meta.getForRead();
-
-        try {
-            BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(buf);
-
-            rootLvl = io.getRootLevel(buf);
-            rootId = io.getLeftmostPageId(buf, rootLvl);
-        }
-        finally {
-            g.meta.releaseRead();
-        }
-
-        g.restartFromRoot(rootId, rootLvl, globalRmvId.get());
-    }
-
-    /**
      * @param g Get.
      */
     private void doFind(Get g) throws IgniteCheckedException {
         try {
             for (;;) { // Go down with retries.
-                initOperation(g);
+                g.initOperation();
 
                 switch (findDown(g, g.rootId, 0L, g.rootLvl)) {
                     case Get.RETRY:
@@ -860,7 +831,7 @@ public abstract class BPlusTree<L, T extends L> {
 
         try {
             for (;;) {
-                initOperation(r);
+                r.initOperation();
 
                 switch (removeDown(r, r.rootId, 0L, 0L, r.rootLvl)) {
                     case Remove.RETRY:
@@ -1039,7 +1010,7 @@ public abstract class BPlusTree<L, T extends L> {
 
         try {
             for (;;) { // Go down with retries.
-                initOperation(p);
+                p.initOperation();
 
                 switch (putDown(p, p.rootId, 0L, p.rootLvl)) {
                     case Put.RETRY:
@@ -1361,13 +1332,13 @@ public abstract class BPlusTree<L, T extends L> {
         /** */
         long rmvId;
 
-        /** Starting point root level. May be outdated. Must be modified only in {@link #initOperation(Get)}. */
+        /** Starting point root level. May be outdated. Must be modified only in {@link Get#initOperation()}. */
         int rootLvl;
 
-        /** Starting point root ID. May be outdated. Must be modified only in {@link #initOperation(Get)}. */
+        /** Starting point root ID. May be outdated. Must be modified only in {@link Get#initOperation()}. */
         long rootId;
 
-        /** Meta page. Initialized by {@link #initOperation(Get)}, released by {@link Get#releaseMeta()}. */
+        /** Meta page. Initialized by {@link Get#initOperation()}, released by {@link Get#releaseMeta()}. */
         Page meta;
 
         /** */
@@ -1392,11 +1363,40 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
+         * Initialize the given operation.
+         *
+         * !!! Symmetrically with this method must be called {@link Get#releaseMeta()} in {@code finally} block.
+         *
+         * @throws IgniteCheckedException If failed.
+         */
+        final void initOperation() throws IgniteCheckedException {
+            if (meta == null)
+                meta = page(metaPageId);
+
+            int rootLvl;
+            long rootId;
+
+            ByteBuffer buf = meta.getForRead();
+
+            try {
+                BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(buf);
+
+                rootLvl = io.getRootLevel(buf);
+                rootId = io.getLeftmostPageId(buf, rootLvl);
+            }
+            finally {
+                meta.releaseRead();
+            }
+
+            restartFromRoot(rootId, rootLvl, globalRmvId.get());
+        }
+
+        /**
          * @param rootId Root page ID.
          * @param rootLvl Root level.
          * @param rmvId Remove ID to be afraid of.
          */
-        void restartFromRoot(long rootId, int rootLvl, long rmvId) {
+        final void restartFromRoot(long rootId, int rootLvl, long rmvId) {
             this.rootId = rootId;
             this.rootLvl = rootLvl;
             this.rmvId = rmvId;


[18/50] ignite git commit: GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.

Posted by sb...@apache.org.
GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.

Review fixes:
- Replace BitSet with HashSet.
- Remove array allocation.
- Replace atomic write to volatile write for timestamp.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/36b0a20c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/36b0a20c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/36b0a20c

Branch: refs/heads/ignite-db-x-10884
Commit: 36b0a20c5b2d9bde2b476ccd7412123b8c2f2255
Parents: 99915a6
Author: dkarachentsev <dk...@gridgain.com>
Authored: Mon Apr 25 10:30:40 2016 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Mon Apr 25 10:30:40 2016 +0300

----------------------------------------------------------------------
 .../internal/pagemem/DirectMemoryUtils.java     |  10 ++
 .../ignite/internal/pagemem/impl/PageImpl.java  |   8 +-
 .../internal/pagemem/impl/PageMemoryImpl.java   | 109 +++++++++----------
 3 files changed, 65 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/36b0a20c/modules/core/src/main/java/org/apache/ignite/internal/pagemem/DirectMemoryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/DirectMemoryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/DirectMemoryUtils.java
index ee9ce08..4c06278 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/DirectMemoryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/DirectMemoryUtils.java
@@ -86,4 +86,14 @@ public class DirectMemoryUtils {
     public boolean compareAndSwapLong(long ptr, long expVal, long newVal) {
         return GridUnsafe.compareAndSwapLong(null, ptr, expVal, newVal);
     }
+
+    /**
+     * Volatile write long to the given memory pointer.
+     *
+     * @param ptr Memory pointer.
+     * @param val Value to write.
+     */
+    public void writeLongVolatile(final long ptr, final long val) {
+        GridUnsafe.putLongVolatile(null, ptr, val);
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/36b0a20c/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
index 6e33572..28d4a19 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
@@ -141,7 +141,7 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
     @Override public ByteBuffer getForRead() {
         acquireShared(1);
 
-        pageMem.atomicWriteCurrentTimestamp(ptr);
+        pageMem.writeCurrentTimestamp(ptr);
 
         return reset(buf.asReadOnlyBuffer());
     }
@@ -158,7 +158,7 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
 
         markDirty();
 
-        pageMem.atomicWriteCurrentTimestamp(ptr);
+        pageMem.writeCurrentTimestamp(ptr);
 
         return reset(buf);
     }
@@ -172,7 +172,7 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
 
             setExclusiveOwnerThread(th);
 
-            pageMem.atomicWriteCurrentTimestamp(ptr);
+            pageMem.writeCurrentTimestamp(ptr);
         }
 
         return reset(buf);
@@ -238,7 +238,7 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
     void acquireReference() {
         refCntUpd.incrementAndGet(this);
 
-        pageMem.atomicWriteCurrentTimestamp(ptr);
+        pageMem.writeCurrentTimestamp(ptr);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/36b0a20c/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
index 5ee8bf3..5075f6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
@@ -25,8 +25,10 @@ import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
@@ -231,7 +233,7 @@ public class  PageMemoryImpl implements PageMemory {
 
             writePageId(absPtr, pageId);
             writePageCacheId(absPtr, cacheId);
-            atomicWriteCurrentTimestamp(absPtr);
+            writeCurrentTimestamp(absPtr);
         }
 
         // TODO pass an argument to decide whether the page should be cleaned.
@@ -334,7 +336,7 @@ public class  PageMemoryImpl implements PageMemory {
                 long absPtr = absolute(relPtr);
 
                 writeFullPageId(absPtr, fullId);
-                atomicWriteCurrentTimestamp(absPtr);
+                writeCurrentTimestamp(absPtr);
 
                 // We can clear dirty flag after the page has been allocated.
                 setDirty(fullId, absPtr, false);
@@ -645,19 +647,21 @@ public class  PageMemoryImpl implements PageMemory {
             dirtyPages.remove(pageId);
     }
 
-    void atomicWriteCurrentTimestamp(final long absPtr) {
-        while (true) {
-            final long readTs = readTimestamp(absPtr);
-
-            if (mem.compareAndSwapLong(absPtr + PAGE_TIMESTAMP_OFFSET, readTs, U.currentTimeMillis()))
-                break;
-        }
-    }
-
-    void writeTimestamp(final long absPtr, final long ts) {
-        mem.writeLong(absPtr + PAGE_TIMESTAMP_OFFSET, ts);
+    /**
+     * Volatile write for current timestamp to page in {@code absAddr} address.
+     *
+     * @param absPtr Absolute page address.
+     */
+    void writeCurrentTimestamp(final long absPtr) {
+        mem.writeLongVolatile(absPtr + PAGE_TIMESTAMP_OFFSET, U.currentTimeMillis());
     }
 
+    /**
+     * Read for timestamp from page in {@code absAddr} address.
+     *
+     * @param absPtr Absolute page address.
+     * @return Timestamp.
+     */
     long readTimestamp(final long absPtr) {
         return mem.readLong(absPtr + PAGE_TIMESTAMP_OFFSET);
     }
@@ -941,8 +945,6 @@ public class  PageMemoryImpl implements PageMemory {
     private long evictPage(final Segment seg) throws IgniteCheckedException {
         final ThreadLocalRandom rnd = ThreadLocalRandom.current();
 
-        final long[] pageRelAddrs = new long[RANDOM_PAGES_EVICT_NUM];
-
         final int cap = seg.loadedPages.capacity();
 
         if (seg.acquiredPages.size() >= seg.loadedPages.size())
@@ -950,24 +952,48 @@ public class  PageMemoryImpl implements PageMemory {
 
         // With big number of random picked pages we may fall into infinite loop, because
         // every time the same page may be found.
-        SimpleLongSet ignored = null;
+        Set<Long> ignored = null;
+
+        long relEvictAddr = INVALID_REL_PTR;
 
         while (true) {
+            long cleanAddr = INVALID_REL_PTR;
+            long cleanTs = Long.MAX_VALUE;
+            long dirtyTs = Long.MAX_VALUE;
+            long dirtyAddr = INVALID_REL_PTR;
+
             for (int i = 0; i < RANDOM_PAGES_EVICT_NUM; i++) {
                 // We need to lookup for pages only in current segment for thread safety,
                 // so peeking random memory will lead to checking for found page segment.
                 // It's much faster to check available pages for segment right away.
-                final long addr = seg.loadedPages.getNearestAt(rnd.nextInt(cap), INVALID_REL_PTR);
+                final long rndAddr = seg.loadedPages.getNearestAt(rnd.nextInt(cap), INVALID_REL_PTR);
 
-                assert addr != INVALID_REL_PTR;
+                assert rndAddr != INVALID_REL_PTR;
 
-                if (ignored != null && ignored.contains(addr))
+                if (relEvictAddr == rndAddr || (ignored != null && ignored.contains(rndAddr))) {
                     i--;
-                else
-                    pageRelAddrs[i] = addr;
-            }
 
-            final long relEvictAddr = findSuitablePageRelAddr(pageRelAddrs);
+                    continue;
+                }
+
+                final long absPageAddr = absolute(rndAddr);
+
+                final long pageTs = readTimestamp(absPageAddr);
+
+                final boolean dirty = isDirty(absPageAddr);
+
+                if (pageTs < cleanTs && !dirty) {
+                    cleanAddr = rndAddr;
+
+                    cleanTs = pageTs;
+                } else if (pageTs < dirtyTs && dirty) {
+                    dirtyAddr = rndAddr;
+
+                    dirtyTs = pageTs;
+                }
+
+                relEvictAddr = cleanAddr == INVALID_REL_PTR ? dirtyAddr : cleanAddr;
+            }
 
             assert relEvictAddr != INVALID_REL_PTR;
 
@@ -979,7 +1005,7 @@ public class  PageMemoryImpl implements PageMemory {
 
             if (fullPageId.pageId() == metaPageId && fullPageId.cacheId() == 0) {
                 if (ignored == null)
-                    ignored = new SimpleLongSet();
+                    ignored = new HashSet<>();
 
                 ignored.add(relEvictAddr);
 
@@ -992,7 +1018,7 @@ public class  PageMemoryImpl implements PageMemory {
                 seg.loadedPages.remove(fullPageId);
             else {
                 if (ignored == null)
-                    ignored = new SimpleLongSet();
+                    ignored = new HashSet<>();
 
                 ignored.add(relEvictAddr);
 
@@ -1011,39 +1037,6 @@ public class  PageMemoryImpl implements PageMemory {
     }
 
     /**
-     * Find oldest and preferable not dirty page from passed ones.
-     *
-     * @param relAddrs Addresses to find from.
-     * @return The oldest and may be dirty page relative address.
-     */
-    private long findSuitablePageRelAddr(final long[] relAddrs) {
-        long addr = INVALID_REL_PTR;
-        long ts = Long.MAX_VALUE;
-        long dirtyTs = Long.MAX_VALUE;
-        long dirtyAddr = INVALID_REL_PTR;
-
-        for (final long relAddr : relAddrs) {
-            final long absPageAddr = absolute(relAddr);
-
-            final long pageTs = readTimestamp(absPageAddr);
-
-            final boolean dirty = isDirty(absPageAddr);
-
-            if (pageTs < ts && !dirty) {
-                addr = relAddr;
-
-                ts = pageTs;
-            } else if (pageTs < dirtyTs && dirty) {
-                dirtyAddr = relAddr;
-
-                dirtyTs = pageTs;
-            }
-        }
-
-        return addr == INVALID_REL_PTR ? dirtyAddr : addr;
-    }
-
-    /**
      * @param relPtr Relative pointer to free.
      */
     private void releaseFreePage(long relPtr) {


[50/50] ignite git commit: Merge remote-tracking branch 'remotes/community/ignite-db-x' into ignite-db-x-10884

Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/community/ignite-db-x' into ignite-db-x-10884

Conflicts:
	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
	modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheH2DatabaseManager.java
	modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
	modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fc7e0630
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fc7e0630
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fc7e0630

Branch: refs/heads/ignite-db-x-10884
Commit: fc7e0630b1c3d4426152f6a77068f3a932523191
Parents: ce4698f 06feabe
Author: sboikov <sb...@gridgain.com>
Authored: Thu Apr 28 11:52:18 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Apr 28 11:52:18 2016 +0300

----------------------------------------------------------------------
 .../mem/unsafe/UnsafeMemoryProvider.java        |    1 +
 .../internal/pagemem/DirectMemoryUtils.java     |   10 +
 .../ignite/internal/pagemem/FullPageId.java     |   44 +-
 .../ignite/internal/pagemem/PageIdUtils.java    |   89 +-
 .../ignite/internal/pagemem/PageMemory.java     |    3 +-
 .../internal/pagemem/impl/FullPageIdTable.java  |  111 +-
 .../ignite/internal/pagemem/impl/PageImpl.java  |   23 +-
 .../internal/pagemem/impl/PageMemoryImpl.java   |  239 +++-
 .../processors/cache/database/MetaStore.java    |   39 +
 .../cache/database/MetadataStorage.java         |   49 +-
 .../processors/cache/database/RowStore.java     |   29 +-
 .../cache/database/freelist/FreeItem.java       |   48 +-
 .../cache/database/freelist/FreeList.java       |  147 +-
 .../cache/database/freelist/FreeTree.java       |   34 +-
 .../cache/database/freelist/io/FreeIO.java      |    8 +-
 .../cache/database/freelist/io/FreeInnerIO.java |   57 +-
 .../cache/database/freelist/io/FreeLeafIO.java  |   41 +-
 .../cache/database/tree/BPlusTree.java          | 1333 ++++++++++--------
 .../cache/database/tree/io/BPlusIO.java         |   23 +-
 .../cache/database/tree/io/BPlusInnerIO.java    |   34 +-
 .../cache/database/tree/io/BPlusLeafIO.java     |    9 +-
 .../cache/database/tree/io/BPlusMetaIO.java     |    8 +-
 .../cache/database/tree/io/DataPageIO.java      |   57 +-
 .../cache/database/tree/io/PageIO.java          |    4 +-
 .../cache/database/tree/reuse/ReuseBag.java     |   35 +
 .../cache/database/tree/reuse/ReuseList.java    |  101 +-
 .../cache/database/tree/reuse/ReuseTree.java    |   10 +-
 .../database/tree/reuse/io/ReuseInnerIO.java    |    6 +-
 .../database/tree/reuse/io/ReuseLeafIO.java     |   10 +-
 .../cache/database/tree/util/PageHandler.java   |    4 +-
 .../pagemem/impl/PageIdUtilsSelfTest.java       |   39 +
 .../processors/query/h2/database/H2Tree.java    |   14 +-
 .../query/h2/database/H2TreeIndex.java          |    5 +-
 .../query/h2/database/io/H2InnerIO.java         |    6 +-
 .../query/h2/database/io/H2LeafIO.java          |   10 +-
 .../database/BPlusTreeReuseSelfTest.java        |   34 +
 .../processors/database/BPlusTreeSelfTest.java  |  713 ++++++++++
 .../IgniteDbMultiNodePutGetSelfTest.java        |    5 +
 .../IgniteDbSingleNodePutGetSelfTest.java       |  123 +-
 .../database/MetadataStorageSelfTest.java       |    4 +-
 40 files changed, 2618 insertions(+), 941 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/fc7e0630/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
index c9ae050,97ea6df..85562de
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
@@@ -53,22 -53,16 +53,22 @@@ public abstract class PageIO 
      public static final short T_H2_REF_INNER = 4;
  
      /** */
 -    public static final short T_FREE_LEAF = 5;
 +    public static final short T_DATA_REF_INNER = 5;
  
      /** */
 -    public static final short T_FREE_INNER = 6;
 +    public static final short T_DATA_REF_LEAF = 6;
  
      /** */
 -    public static final short T_REUSE_LEAF = 7;
 +    public static final short T_FREE_LEAF = 7;
  
      /** */
 -    public static final short T_REUSE_INNER = 8;
 +    public static final short T_FREE_INNER = 8;
 +
 +    /** */
 +    public static final short T_REUSE_LEAF = 9;
 +
 +    /** */
-     public static final short T_REUSE_INNER = 8;
++    public static final short T_REUSE_INNER = 10;
  
      /** */
      private final int ver;

http://git-wip-us.apache.org/repos/asf/ignite/blob/fc7e0630/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ignite/blob/fc7e0630/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
index f83d050,99dc937..c50d012
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
@@@ -15,8 -15,8 +15,9 @@@ import java.util.List
  import java.util.Map;
  import java.util.Random;
  import org.apache.ignite.IgniteCache;
+ import org.apache.ignite.IgniteDataStreamer;
  import org.apache.ignite.cache.CacheAtomicityMode;
 +import org.apache.ignite.cache.CachePeekMode;
  import org.apache.ignite.cache.CacheRebalanceMode;
  import org.apache.ignite.cache.CacheWriteSynchronizationMode;
  import org.apache.ignite.cache.query.SqlFieldsQuery;
@@@ -275,40 -274,120 +276,157 @@@ public class IgniteDbSingleNodePutGetSe
      /**
       * @throws Exception if failed.
       */
 +    public void testSizeClear() throws Exception {
 +        IgniteEx ig = grid(0);
 +
 +        final IgniteCache<Integer, DbValue> cache = ig.cache(null);
 +
 +        GridCacheAdapter<Object, Object> internalCache = ig.context().cache().internalCache();
 +
 +        int cnt = 5000;
 +
 +        X.println("Put start");
 +
 +        for (int i = 0; i < cnt; i++) {
 +            DbValue v0 = new DbValue(i, "test-value", i);
 +
 +            cache.put(i, v0);
 +
 +            checkEmpty(internalCache, i);
 +
 +            assertEquals(v0, cache.get(i));
 +        }
 +
 +        awaitPartitionMapExchange();
 +
 +        assertEquals(cnt, cache.size(CachePeekMode.OFFHEAP));
 +
 +        X.println("Clear start.");
 +
 +        cache.clear();
 +
 +        assertEquals(0, cache.size(CachePeekMode.OFFHEAP));
 +
 +        for (int i = 0; i < cnt; i++)
 +            assertNull(cache.get(i));
 +    }
++    /**
++     * @throws Exception if failed.
++     */
+     public void testBounds() throws Exception {
+         IgniteEx ig = grid(0);
+ 
+         final IgniteCache<Integer, DbValue> cache = ig.cache(null);
+ 
+         X.println("Put start");
+ 
+         int cnt = 1000;
+ 
+         try (IgniteDataStreamer<Integer, DbValue> st = ig.dataStreamer(null)) {
+             st.allowOverwrite(true);
+ 
+             for (int i = 0; i < cnt; i++) {
+                 int k = 2 * i;
+ 
+                 DbValue v0 = new DbValue(k, "test-value", k);
+ 
+                 st.addData(k, v0);
+             }
+         }
+ 
+         X.println("Get start");
+ 
+         for (int i = 0; i < cnt; i++) {
+             int k = 2 * i;
+ 
+             DbValue v0 = new DbValue(k, "test-value", k);
+ 
+             assertEquals(v0, cache.get(k));
+         }
+ 
+         awaitPartitionMapExchange();
+ 
+         X.println("Query start");
+ 
+         // Make sure to cover multiple pages.
+         int limit = 500;
+ 
+         for (int i = 0; i < limit; i++) {
+             List<List<?>> res = cache.query(new SqlFieldsQuery("select ival, _val from dbvalue where ival < ? order by ival")
+                 .setArgs(i)).getAll();
+ 
+             // 0 => 0, 1 => 1, 2=>1,...
+             assertEquals((i + 1) / 2, res.size());
+ 
+             res = cache.query(new SqlFieldsQuery("select ival, _val from dbvalue where ival <= ? order by ival")
+                 .setArgs(i)).getAll();
+ 
+             // 0 => 1, 1 => 1, 2=>2,...
+             assertEquals(i / 2 + 1, res.size());
+         }
+     }
+ 
+     /**
+      * @throws Exception if failed.
+      */
+     public void testMultithreadedPut() throws Exception {
+         IgniteEx ig = grid(0);
+ 
+         final IgniteCache<Integer, DbValue> cache = ig.cache(null);
+ 
+         X.println("Put start");
+ 
+         int cnt = 20_000;
+ 
+         try (IgniteDataStreamer<Integer, DbValue> st = ig.dataStreamer(null)) {
+             st.allowOverwrite(true);
+ 
+             for (int i = 0; i < cnt; i++) {
+                 DbValue v0 = new DbValue(i, "test-value", i);
+ 
+                 st.addData(i, v0);
+             }
+         }
+ 
+         X.println("Get start");
+ 
+         for (int i = 0; i < cnt; i++) {
+             DbValue v0 = new DbValue(i, "test-value", i);
+ 
+             assertEquals(v0, cache.get(i));
+         }
+ 
+         awaitPartitionMapExchange();
+ 
+         X.println("Query start");
+ 
+         assertEquals(cnt, cache.query(new SqlFieldsQuery("select null from dbvalue")).getAll().size());
+ 
+         int limit = 500;
+ 
+         List<List<?>> res = cache.query(new SqlFieldsQuery("select ival, _val from dbvalue where ival < ? order by ival")
+             .setArgs(limit)).getAll();
+ 
+         assertEquals(limit, res.size());
+ 
+         for (int i = 0; i < limit; i++) {
+             List<?> row = res.get(i);
+ 
+             assertEquals(2, row.size());
+             assertEquals(i, row.get(0));
+ 
+             assertEquals(new DbValue(i, "test-value", i), row.get(1));
+         }
+ 
+         assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
+         assertEquals(2000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 7000"))
+             .getAll().size());
+ 
+         String plan = cache.query(new SqlFieldsQuery(
+             "explain select lval from dbvalue where ival >= 5000 and ival < 7000")).getAll().get(0).get(0).toString();
+ 
+         assertTrue(plan, plan.contains("iVal_idx"));
+     }
  
      /**
       * @throws Exception if failed.


[36/50] ignite git commit: ignite-db - size + minor

Posted by sb...@apache.org.
ignite-db - size + minor


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c523c6fa
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c523c6fa
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c523c6fa

Branch: refs/heads/ignite-db-x-10884
Commit: c523c6fac50a7f6a56e66504da0a096e6f7e3a44
Parents: 2b30543
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 02:06:18 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 02:06:18 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 55 +++++++++++++++++---
 .../processors/database/BPlusTreeSelfTest.java  |  9 +---
 2 files changed, 49 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/c523c6fa/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 70f1af5..e712a41 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -577,7 +577,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @param upper Upper bound.
      * @return Cursor.
      */
-    private GridCursor<T> findNoLower(L upper) throws IgniteCheckedException {
+    private GridCursor<T> findLowerUnbounded(L upper) throws IgniteCheckedException {
         ForwardCursor cursor = new ForwardCursor(upper);
 
         long firstPageId;
@@ -608,7 +608,7 @@ public abstract class BPlusTree<L, T extends L> {
      */
     public final GridCursor<T> find(L lower, L upper) throws IgniteCheckedException {
         if (lower == null)
-            return findNoLower(upper);
+            return findLowerUnbounded(upper);
 
         GetCursor g = new GetCursor(lower, upper);
 
@@ -996,11 +996,52 @@ public abstract class BPlusTree<L, T extends L> {
      * @param max Max.
      * @return Random value from {@code 0} (inclusive) to the given max value (exclusive).
      */
-     public int randomInt(int max) {
+    public int randomInt(int max) {
          return ThreadLocalRandom.current().nextInt(max);
      }
 
     /**
+     * TODO may produce wrong results on concurrent access
+     *
+     * @return Size.
+     * @throws IgniteCheckedException If failed.
+     */
+    public final long size() throws IgniteCheckedException {
+        long cnt = 0;
+        long pageId;
+
+        try (Page meta = page(metaPageId)) {
+            pageId = getFirstPageId(meta, 0); // Level 0 is always at the bottom.
+        }
+
+        BPlusIO<L> io = null;
+
+        while (pageId != 0) {
+            try (Page page = page(pageId)) {
+                ByteBuffer buf = page.getForRead();
+
+                try {
+                    if (io == null) {
+                        io = io(buf);
+
+                        assert io.isLeaf();
+                    }
+
+                    cnt += io.getCount(buf);
+
+
+                    pageId = io.getForward(buf);
+                }
+                finally {
+                    page.releaseRead();
+                }
+            }
+        }
+
+        return cnt;
+    }
+
+    /**
      * @param row Row.
      * @return Old row.
      * @throws IgniteCheckedException If failed.
@@ -1708,9 +1749,9 @@ public abstract class BPlusTree<L, T extends L> {
         /**
          * @param t Tail.
          * @return {@code true} If merged successfully or end reached.
-         * @throws IgniteCheckedException
+         * @throws IgniteCheckedException If failed.
          */
-        private boolean mergeDown(Tail<L> t) throws IgniteCheckedException {
+        private boolean mergeBottomUp(Tail<L> t) throws IgniteCheckedException {
             assert needMergeEmptyBranch == FALSE || needMergeEmptyBranch == DONE: needMergeEmptyBranch;
 
             if (t.down == null)
@@ -1719,7 +1760,7 @@ public abstract class BPlusTree<L, T extends L> {
             if (t.down.sibling == null) // We've merged something there.
                 return false;
 
-            return mergeDown(t.down) && merge(t);
+            return mergeBottomUp(t.down) && merge(t);
         }
 
         /**
@@ -1745,7 +1786,7 @@ public abstract class BPlusTree<L, T extends L> {
                 needMergeEmptyBranch = DONE;
             }
 
-            mergeDown(tail);
+            mergeBottomUp(tail);
 
             if (needReplaceInner == READY) {
                 // If we've merged empty branch right now, then the inner key was dropped.

http://git-wip-us.apache.org/repos/asf/ignite/blob/c523c6fa/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index eebd304..813fdc3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -475,9 +475,6 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
             if (i % 1000 == 0)
                 X.println(" --> " + i + "  " + x);
 
-//            if (i >= 57)
-//                X.println(tree.printTree());
-
             if (tree.randomInt(2) == 0)
                 assertEquals(map.put(x, x), tree.put(x));
             else {
@@ -489,11 +486,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
             if (i % 100 == 0) {
                 GridCursor<Long> cursor = tree.find(null, null);
 
-                int size = 0;
-
                 while (cursor.next()) {
-                    size++;
-
                     x = cursor.get();
 
                     assert x != null;
@@ -501,7 +494,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
                     assertEquals(map.get(x), x);
                 }
 
-                assertEquals(map.size(), size);
+                assertEquals(map.size(), tree.size());
             }
         }
     }


[04/50] ignite git commit: ignite-db - reuse list

Posted by sb...@apache.org.
ignite-db - reuse list


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1ec6712e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1ec6712e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1ec6712e

Branch: refs/heads/ignite-db-x-10884
Commit: 1ec6712ec7b21a2f4e35d5e587971ba8642febd9
Parents: 813c36f
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 20 23:16:33 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 20 23:16:33 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/reuse/ReuseList.java    | 24 ++++++++------------
 1 file changed, 9 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/1ec6712e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
index 186fee5..a40fa18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.processors.cache.database.tree.reuse;
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.processors.cache.database.MetaStore;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
@@ -36,9 +35,6 @@ public class ReuseList {
     /** */
     private final ReuseTree[] trees;
 
-    /** */
-    private boolean ready;
-
     /**
      * @param cacheId Cache ID.
      * @param pageMem Page memory.
@@ -49,17 +45,18 @@ public class ReuseList {
     public ReuseList(int cacheId, PageMemory pageMem, int segments, MetaStore metaStore) throws IgniteCheckedException {
         A.ensure(segments > 1, "Segments must be greater than 1.");
 
-        trees = new ReuseTree[segments];
+        ReuseTree[] trees0 = new ReuseTree[segments];
 
-        for (int i = 0; i < trees.length; i++) {
+        for (int i = 0; i < segments; i++) {
             String idxName = i + "##" + cacheId + "_reuse";
 
             IgniteBiTuple<FullPageId,Boolean> t = metaStore.getOrAllocateForIndex(cacheId, idxName);
 
-            trees[i] = new ReuseTree(this, cacheId, pageMem, t.get1(), t.get2());
+            trees0[i] = new ReuseTree(this, cacheId, pageMem, t.get1(), t.get2());
         }
 
-        ready = true;
+        // Later assignment is done intentionally, see null check in method take.
+        trees = trees0;
     }
 
     /**
@@ -75,9 +72,9 @@ public class ReuseList {
 
         assert tree != null;
 
-        // Avoid dead locks.
+        // Avoid recursion on the same tree.
         if (tree == client) {
-            treeIdx++; // Go forward.
+            treeIdx++; // Go forward and take the next tree.
 
             if (treeIdx == trees.length)
                 treeIdx = 0;
@@ -94,10 +91,8 @@ public class ReuseList {
      * @throws IgniteCheckedException If failed.
      */
     public FullPageId take(BPlusTree<?,?> client) throws IgniteCheckedException {
-        assert PageIdUtils.pageIdx(MIN.pageId()) == 0;
-
-        // Remove and return page at min position.
-        return ready ? tree(client).removeCeil(MIN) : null;
+        // Remove and return page at min possible position.
+        return trees == null ? null : tree(client).removeCeil(MIN);
     }
 
     /**
@@ -106,7 +101,6 @@ public class ReuseList {
      */
     public void put(BPlusTree<?,?> client, FullPageId fullPageId) throws IgniteCheckedException {
         assert fullPageId != null;
-        assert ready;
 
         tree(client).put(fullPageId);
     }


[30/50] ignite git commit: ignite-db - fixes

Posted by sb...@apache.org.
ignite-db - fixes


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3166aeb5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3166aeb5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3166aeb5

Branch: refs/heads/ignite-db-x-10884
Commit: 3166aeb5b7661f1207d825b198e2fd54d87bcb56
Parents: b0cab0c
Author: S.Vladykin <sv...@gridgain.com>
Authored: Tue Apr 26 07:59:32 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Tue Apr 26 07:59:32 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 19 ++--
 .../processors/database/BPlusTreeSelfTest.java  | 99 +++++++++++++++++++-
 2 files changed, 104 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/3166aeb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 6173d72..b90c5b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -1762,8 +1762,9 @@ public abstract class BPlusTree<L, T extends L> {
                 boolean mergeMore = mergeDown(tail);
 
                 if (needReplaceInner == READY) {
-                    // Need to replace inner key with new max key for the left subtree.
-                    replaceInner(needMergeEmptyBranch == DONE);
+                    // If we've merged empty branch then the inner key was dropped.
+                    if (needMergeEmptyBranch != DONE)
+                        replaceInner(); // Need to replace inner key with new max key for the left subtree.
 
                     needReplaceInner = DONE;
                 }
@@ -2007,10 +2008,9 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
-         * @param emptyBranchMerged If empty branch was merged.
          * @throws IgniteCheckedException If failed.
          */
-        private void replaceInner(boolean emptyBranchMerged) throws IgniteCheckedException {
+        private void replaceInner() throws IgniteCheckedException {
             assert needReplaceInner == READY: needReplaceInner;
             assert tail.lvl > 0: "leaf";
             assert innerIdx >= 0: innerIdx;
@@ -2025,21 +2025,14 @@ public abstract class BPlusTree<L, T extends L> {
 
             assert leafCnt > 0: leafCnt; // Leaf must be merged at this point already if it was empty.
 
-            if (innerIdx == innerCnt) {
-                if (emptyBranchMerged)
-                    return; // Our inner key was dropped.
-
-                throw new IllegalStateException();
-            }
-
-            if (innerIdx < inner.getCount()) {
+            if (innerIdx < innerCnt) {
                 // Update inner key with the new biggest key of left subtree.
                 inner.io.store(inner.buf, innerIdx, leaf.io, leaf.buf, leafCnt - 1);
                 leaf.io.setRemoveId(leaf.buf, globalRmvId.get());
             }
             else {
                 // If after leaf merge parent have lost inner key, we don't need to update it anymore.
-                assert innerIdx == inner.getCount();
+                assert innerIdx == innerCnt;
                 assert inner(inner.io).getLeft(inner.buf, innerIdx) == leaf.page.id();
             }
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/3166aeb5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index ce3840a..c2b7a0b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -207,6 +207,103 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
         doTestPutRemove(false);
     }
 
+    // ------- 2 - 40
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_mm_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = -1;
+        RMV_INC = -1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_mm_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = -1;
+        RMV_INC = -1;
+
+        doTestPutRemove(false);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_pm_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = 1;
+        RMV_INC = -1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_pm_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = 1;
+        RMV_INC = -1;
+
+        doTestPutRemove(false);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_pp_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = 1;
+        RMV_INC = 1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_pp_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = 1;
+        RMV_INC = 1;
+
+        doTestPutRemove(false);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_mp_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = -1;
+        RMV_INC = 1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_2_40_mp_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 2;
+        CNT = 40;
+        PUT_INC = -1;
+        RMV_INC = 1;
+
+        doTestPutRemove(false);
+    }
+
     /**
      * @param canGetRow Can get row from inner page.
      * @throws IgniteCheckedException If failed.
@@ -214,7 +311,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     private void doTestPutRemove(boolean canGetRow) throws IgniteCheckedException {
         TestTree tree = createTestTree(canGetRow);
 
-        long cnt = 10;
+        long cnt = CNT;
 
         for (long x = PUT_INC > 0 ? 0 : cnt - 1; x >= 0 && x < cnt; x += PUT_INC) {
             assertNull(tree.findOne(x));


[47/50] ignite git commit: Merge branch 'ignite-db-2' of https://github.com/gridgain/apache-ignite into ignite-db-2

Posted by sb...@apache.org.
Merge branch 'ignite-db-2' of https://github.com/gridgain/apache-ignite into ignite-db-2


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/788b6e8a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/788b6e8a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/788b6e8a

Branch: refs/heads/ignite-db-x-10884
Commit: 788b6e8a2ffd31b69613a6a6ecf61f5f518c32b0
Parents: 5761012 cf5ad2b
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 10:50:16 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 10:50:16 2016 +0300

----------------------------------------------------------------------
 .../internal/pagemem/impl/PageMemoryImpl.java     | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------



[13/50] ignite git commit: ignite-db - wip

Posted by sb...@apache.org.
ignite-db - wip


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/939b883a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/939b883a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/939b883a

Branch: refs/heads/ignite-db-x-10884
Commit: 939b883ab11b1a791bb8056266d91053b42b91e4
Parents: fdd0939
Author: S.Vladykin <sv...@gridgain.com>
Authored: Sun Apr 24 18:57:57 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Sun Apr 24 18:57:57 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 309 ++++++++-----------
 .../cache/database/tree/io/BPlusInnerIO.java    |   2 +-
 .../cache/database/tree/io/BPlusLeafIO.java     |   2 +-
 3 files changed, 130 insertions(+), 183 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/939b883a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index a822923..58f07e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -324,11 +324,11 @@ public abstract class BPlusTree<L, T extends L> {
             if (r.needReplaceInner == TRUE ||
                 // We need to make sure that we have back or forward to be able to merge.
                 ((r.fwdId != 0 || r.backId != 0) && mayMerge(cnt - 1, io.getMaxCount(buf)))) {
-                r.addTail(leaf, buf, io, 0, false, Tail.PRIMARY, Integer.MIN_VALUE);
-
-                if (r.fwdId != 0 && r.backId == 0) // If we have backId then we'll lock back page, nothing to do here.
+                // If we have backId then we've already locked back page, nothing to do here.
+                if (r.fwdId != 0 && r.backId == 0)
                     r.lockForward(0);
 
+                r.addTail(leaf, buf, io, 0, Tail.EXACT, Integer.MIN_VALUE);
 
                 if (r.needReplaceInner == FALSE)
                     r.needMerge = TRUE;
@@ -353,7 +353,7 @@ public abstract class BPlusTree<L, T extends L> {
             if (res == Remove.FOUND && (r.needMerge == TRUE || r.needReplaceInner == TRUE)) {
                 assert r.needMerge == TRUE ^ r.needReplaceInner == TRUE: "we can do only one thing at once";
 
-                r.addTail(back, buf, io, lvl, true, Integer.MIN_VALUE);
+                r.addTail(back, buf, io, lvl, Tail.BACK, Integer.MIN_VALUE);
             }
 
             return res;
@@ -372,7 +372,7 @@ public abstract class BPlusTree<L, T extends L> {
             int res = r.doLockTail(lvl);
 
             if (res == Remove.FOUND)
-                r.addTail(back, buf, io, lvl, true, false, Integer.MIN_VALUE);
+                r.addTail(back, buf, io, lvl, Tail.BACK, Integer.MIN_VALUE);
 
             return res;
         }
@@ -383,7 +383,7 @@ public abstract class BPlusTree<L, T extends L> {
         @Override protected int run0(Page page, ByteBuffer buf, BPlusIO<L> io, Remove r, int lvl)
             throws IgniteCheckedException {
 
-            r.addTail(page, buf, io, lvl, false, false, Integer.MIN_VALUE);
+            r.addTail(page, buf, io, lvl, Tail.FORWARD, Integer.MIN_VALUE);
 
             return Remove.FOUND;
         }
@@ -401,7 +401,7 @@ public abstract class BPlusTree<L, T extends L> {
             boolean found = idx >= 0;
 
             if (found) {
-                // We could not miss the inner value on down move because of `triangle` invariant, thus it must be TRUE.
+                // We can not miss the inner value on down move because of `triangle` invariant, thus it must be TRUE.
                 assert r.needReplaceInner == TRUE: r.needReplaceInner;
                 assert idx <= Short.MAX_VALUE : idx;
 
@@ -418,7 +418,7 @@ public abstract class BPlusTree<L, T extends L> {
             }
 
             // Check that we have a correct view of the world.
-            if (lvl != 0 && inner(io).getLeft(buf, idx) != r.getTail(lvl - 1, false).page.id()) {
+            if (lvl != 0 && inner(io).getLeft(buf, idx) != r.getTail(lvl - 1).page.id()) {
                 assert !found;
 
                 return Remove.RETRY;
@@ -426,16 +426,16 @@ public abstract class BPlusTree<L, T extends L> {
 
             // We don't have a back page, need to lock our forward and become a back for it.
             // If found then we are on inner replacement page, it will be a top parent, no need to lock forward.
-            boolean back = !found && r.fwdId != 0 && r.backId == 0;
+            boolean noBack = !found && r.fwdId != 0 && r.backId == 0;
 
-            if (back)
+            if (noBack)
                 r.lockForward(lvl);
 
-            r.addTail(page, buf, io, lvl, back, true, idx);
+            r.addTail(page, buf, io, lvl, Tail.EXACT, idx);
 
             if (r.needMerge == TRUE) {
                 assert r.needReplaceInner == FALSE;
-                assert r.tail.down != null;
+                assert r.tail.down.type == Tail.EXACT;
 
                 r.needMerge = READY;
             }
@@ -445,22 +445,6 @@ public abstract class BPlusTree<L, T extends L> {
     };
 
     /** */
-    private final PageHandler<Remove> mergePages = new GetPageHandler<Remove>() {
-        @Override protected int run0(Page fwd, ByteBuffer fwdBuf, BPlusIO<L> io, Remove r, int lvl)
-            throws IgniteCheckedException {
-            Tail<L> prnt = r.getTail(lvl + 1, false);
-
-            assert prnt != null;
-
-            Tail<L> t = r.getTail(lvl, false);
-
-            assert t.io == io : "must be the same"; // Otherwise can be not compatible.
-
-            return r.mergePages(prnt, t, fwd, fwdBuf) ? TRUE : FALSE;
-        }
-    };
-
-    /** */
     private final PageHandler<Long> updateLeftmost = new PageHandler<Long>() {
         @Override public int run(Page page, ByteBuffer buf, Long pageId, int lvl) throws IgniteCheckedException {
             assert pageId != null;
@@ -1241,25 +1225,6 @@ public abstract class BPlusTree<L, T extends L> {
     }
 
     /**
-     * @param page Page.
-     * @return Number of row links in the given index page.
-     */
-    private int getLinksCount(Page page) throws IgniteCheckedException {
-        assert page != null;
-
-        ByteBuffer buf = page.getForRead();
-
-        try {
-            BPlusIO<L> io = io(buf);
-
-            return io.getCount(buf);
-        }
-        finally {
-            page.releaseRead();
-        }
-    }
-
-    /**
      * @param p Put.
      * @param pageId Page ID.
      * @param fwdId Expected forward page ID.
@@ -1698,23 +1663,23 @@ public abstract class BPlusTree<L, T extends L> {
         /**
          * Process tail and finish.
          *
-         * @param skipMergeMore Ignore the attempt to merge more pages up.
+         * @param ignoreMergeMore Ignore the attempt to merge more pages up.
          * @return {@code false} If failed to finish and we need to lock more pages up.
          * @throws IgniteCheckedException If failed.
          */
-        private boolean finishTail(boolean skipMergeMore) throws IgniteCheckedException {
+        private boolean finishTail(boolean ignoreMergeMore) throws IgniteCheckedException {
             assert !isFinished();
             assert needMerge != FALSE || needReplaceInner != FALSE;
             assert tail != null;
 
             if (needReplaceInner == READY) {
                 assert needMerge == FALSE: needMerge;
-                assert getTail(0, false) != null: "we must keep lock on the leaf page";
+                assert getTail(0) != null: "we must keep lock on the leaf page";
 
                 // We increment remove ID in write lock on leaf page, thus it is guaranteed that
                 // any successor will get greater value than he had read at the beginning of the operation.
                 // Thus it will be guaranteed to do a retry from root.
-                globalRmvId.incrementAndGet();
+                globalRmvId.incrementAndGet(); // TODO replace with pageId bit patching
 
                 // Need to replace inner key with new max key for the left subtree.
                 doReplaceInner();
@@ -1723,11 +1688,11 @@ public abstract class BPlusTree<L, T extends L> {
             }
             else if (needMerge == READY) {
                 assert needReplaceInner == FALSE: needReplaceInner;
-                assert tail.down != null || tail.fwd.down != null;
+                assert tail.down != null;
 
-                boolean needMergeMore = merge(tail.lvl - 1, true, true);
+                boolean needMergeMore = merge(tail.lvl - 1, true);
 
-                if (needMergeMore && !skipMergeMore) {
+                if (needMergeMore && !ignoreMergeMore) {
                     needMerge = TRUE;
 
                     return false;
@@ -1759,7 +1724,7 @@ public abstract class BPlusTree<L, T extends L> {
             this.fwdId = fwdId;
 
             if (backId == 0)
-                return doRemoveFromLeaf(); // Fast path.
+                return doRemoveFromLeaf();
 
             // Lock back page before the remove, we'll need it for merges.
             Page back = page(backId);
@@ -1831,13 +1796,13 @@ public abstract class BPlusTree<L, T extends L> {
          * @throws IgniteCheckedException If failed.
          */
         private void lockForward(int lvl) throws IgniteCheckedException {
-            assert needReplaceInner == TRUE: needReplaceInner;
             assert fwdId != 0;
             assert backId == 0;
 
             int res = writePage(page(fwdId), lockTailForward, this, lvl, Remove.RETRY);
 
-            assert res == Remove.FOUND;
+            // Must always be called from lock on back page, thus we should never fail here.
+            assert res == Remove.FOUND: res;
         }
 
         /**
@@ -1849,8 +1814,8 @@ public abstract class BPlusTree<L, T extends L> {
          * @param kickLeftChild If we are dropping left child instead of the right one.
          * @throws IgniteCheckedException If failed.
          */
-        private void doRemove(BPlusIO io, Page page, ByteBuffer buf, int cnt, int idx, int lvl,
-            boolean kickLeftChild) throws IgniteCheckedException {
+        private void doRemove(BPlusIO io, Page page, ByteBuffer buf, int cnt, int idx, int lvl, boolean kickLeftChild)
+            throws IgniteCheckedException {
             assert cnt > 0;
             assert idx >= 0;
             assert idx <= cnt;
@@ -1873,16 +1838,14 @@ public abstract class BPlusTree<L, T extends L> {
         /**
          * @param prnt Parent tail.
          * @param cur Current tail.
-         * @param fwd Forward page.
-         * @param fwdBuf Forward buffer.
+         * @param fwd Forward tail.
          * @throws IgniteCheckedException If failed.
          */
-        private boolean mergePages(Tail<L> prnt, Tail<L> cur, Page fwd, ByteBuffer fwdBuf)
-            throws IgniteCheckedException {
-            assert io(fwdBuf) == cur.io;
+        private boolean mergePages(Tail<L> prnt, Tail<L> cur, Tail<L> fwd) throws IgniteCheckedException {
+            assert fwd.io == cur.io; // Otherwise incompatible.
 
             int cnt = cur.io.getCount(cur.buf);
-            int fwdCnt = cur.io.getCount(fwdBuf);
+            int fwdCnt = fwd.io.getCount(fwd.buf);
             int newCnt = countAfterMerge(cur, fwdCnt);
 
             if (newCnt == -1) // Not enough space.
@@ -1906,8 +1869,8 @@ public abstract class BPlusTree<L, T extends L> {
                 cnt++;
             }
 
-            cur.io.copyItems(fwdBuf, cur.buf, 0, cnt, fwdCnt, true);
-            cur.io.setForward(cur.buf, cur.io.getForward(fwdBuf));
+            cur.io.copyItems(fwd.buf, cur.buf, 0, cnt, fwdCnt, true);
+            cur.io.setForward(cur.buf, fwd.io.getForward(fwd.buf));
 
             assert prntCnt > 0: prntCnt;
 
@@ -1915,7 +1878,7 @@ public abstract class BPlusTree<L, T extends L> {
             doRemove(prnt.io, prnt.page, prnt.buf, prntCnt, prnt.idx, prnt.lvl, false);
 
             // Forward page is now empty and has no links.
-            freePage(fwd, fwdBuf, cur.io, cur.lvl);
+            freePage(fwd.page, fwd.buf, fwd.io, fwd.lvl);
 
             return true;
         }
@@ -1967,7 +1930,6 @@ public abstract class BPlusTree<L, T extends L> {
             int cnt = inner.io.getCount(inner.buf);
 
             assert cnt > 0: cnt;
-            assert inner.fwd == null: "if we've found our inner key in this page it can't be a back page";
 
             // We need to check if the branch we are going to drop goes to the left or to the right.
             boolean kickLeft = inner.down.page.id() == inner(inner.io).getLeft(inner.buf, inner.idx);
@@ -1981,8 +1943,7 @@ public abstract class BPlusTree<L, T extends L> {
             // Otherwise we can be sure that inner page was not freed, at lead it must become
             // an empty routing page. Thus always starting from inner.down here.
             for (Tail t = inner.down; t != null; t = t.down) {
-                if (t.fwd != null)
-                    t = t.fwd;
+                // TODO merge branch but not just free it
 
                 assert t.io.getCount(t.buf) == 0: row;
 
@@ -1998,15 +1959,16 @@ public abstract class BPlusTree<L, T extends L> {
             assert tail.lvl > 0;
             assert innerIdx >= 0;
 
-            Tail<L> leaf = getTail(0, false);
-            Tail<L> inner = getTail(tail.lvl, false);
+            Tail<L> leaf = getTail(0);
+            Tail<L> inner = tail;
 
+            assert inner.type == Tail.EXACT: inner.type;
             assert innerIdx < inner.io.getCount(inner.buf);
 
             int cnt = leaf.io.getCount(leaf.buf);
 
             if (cnt == 0) { // Merge empty leaf page.
-                if (!merge(0, true, false)) {
+                if (!merge(0, false)) {
                     // For leaf pages this can happen only when parent is empty -> drop the whole branch.
                     dropEmptyBranch(inner);
 
@@ -2014,7 +1976,7 @@ public abstract class BPlusTree<L, T extends L> {
                 }
 
                 // Need to handle possible tail restructuring after merge.
-                leaf = getTail(0, false);
+                leaf = getTail(0);
 
                 cnt = leaf.io.getCount(leaf.buf);
 
@@ -2038,84 +2000,59 @@ public abstract class BPlusTree<L, T extends L> {
 
         /**
          * @param lvl Level.
-         * @param skipMayMerge Skip checking for {@link #mayMerge(int, int)}.
-         * @return {@code true} If merged, {@code false} if not (for example because of insufficient space).
+         * @return {@code true} If merged, {@code false} if not (because of insufficient space).
          * @throws IgniteCheckedException If failed.
          */
-        private boolean merge(int lvl, boolean skipMayMerge, boolean releaseMerged) throws IgniteCheckedException {
+        private boolean merge(int lvl, boolean releaseMerged) throws IgniteCheckedException {
             assert tail.lvl > lvl;
 
-            Tail<L> prnt = getTail(lvl + 1, false);
+            Tail<L> prnt = getTail(lvl + 1);
 
             if (prnt.io.getCount(prnt.buf) == 0)
-                return false; // Parent is an empty routing page, forward page will have another parent.
-
-            Tail<L> cur = getTail(lvl, false);
-            Tail<L> back = getTail(lvl, true);
+                return false; // Parent is an empty routing page, child forward page will have another parent.
 
-            if (back == null) {
-                // We don't have a back page -> last move was to the left.
-                long fwdId = cur.io.getForward(cur.buf);
+            Tail<L> cur = getTail(lvl);
+            Tail<L> back = cur.sibling;
 
-                if (fwdId == 0) // We can get 0 only in the last rightmost page with empty routing parent -> drop both.
-                    return false;
-
-                int cnt = cur.io.getCount(cur.buf);
-
-                if (!skipMayMerge) {
-                    int maxCnt = cur.io.getMaxCount(cur.buf);
+            assert prnt.down == cur : prnt.down;
+            assert back != null: "we must have a partner to merge with";
 
-                    if (!mayMerge(cnt, maxCnt))
-                        return false;
-                }
+            if (back.type != Tail.BACK) { // Flip if it was actually FORWARD but not BACK.
+                assert back.type == Tail.FORWARD: back.type;
 
-                try (Page fwd = page(fwdId)) {
-                    assert fwd != null; // We've locked cur page which is back for fwd.
-
-                    // Check count in read lock first.
-                    int fwdCnt = getLinksCount(fwd);
-
-                    if (countAfterMerge(cur, fwdCnt) == -1)
-                        return false;
+                back = cur; // Current goes back.
+                cur = cur.sibling; // Forward becomes current.
+            }
 
-                    if (writePage(fwd, mergePages, this, lvl, FALSE) == TRUE) {
-                        if (releaseMerged) {
-                            prnt.down = null;
+            assert cur.io == back.io: "must always be the same"; // Otherwise can be not compatible.
 
-                            writeUnlockAndClose(cur.page);
-                        }
+            if (!mergePages(prnt, back, cur))
+                return false;
 
-                        return true;
-                    }
+            // BACK becomes EXACT.
+            if (back.type == Tail.BACK) {
+                assert back.sibling == null;
 
-                    return false;
-                }
+                back.down = cur.down;
+                back.type = Tail.EXACT;
+                prnt.down = back;
             }
-            else {
-                assert cur.io == back.io: "must always be the same"; // Otherwise can be not compatible.
-
-                if (mergePages(prnt, back, cur.page, cur.buf)) {
-                    assert prnt.down == back;
-                    assert back.fwd == cur;
-
-                    // Back becomes current, current is dropped.
-                    back.down = cur.down;
-                    back.fwd = null;
+            else { // back is already EXACT
+                assert back.type == Tail.EXACT: back.type;
 
-                    // Always unlock and release current because we made it invisible for further code.
-                    writeUnlockAndClose(cur.page);
+                back.sibling = null;
+            }
 
-                    if (releaseMerged) {
-                        prnt.down = null;
+            // Always unlock and release current because we made it invisible for further code.
+            writeUnlockAndClose(cur.page);
 
-                        writeUnlockAndClose(back.page);
-                    }
+            if (releaseMerged) {
+                prnt.down = null;
 
-                    return true;
-                }
-
-                return false;
+                writeUnlockAndClose(back.page);
             }
+
+            return true;
         }
 
         /**
@@ -2136,10 +2073,10 @@ public abstract class BPlusTree<L, T extends L> {
             while (t != null) {
                 writeUnlockAndClose(t.page);
 
-                if (t.down != null)
-                    t = t.down;
-                else
-                    t = t.fwd;
+                if (t.sibling != null)
+                    writeUnlockAndClose(t.sibling.page);
+
+                t = t.down;
             }
         }
 
@@ -2160,13 +2097,16 @@ public abstract class BPlusTree<L, T extends L> {
                 if (t.lvl < lvl)
                     return false;
 
-                if (t.lvl == lvl && t.page.id() == pageId)
-                    return true;
+                if (t.lvl == lvl) {
+                    if (t.page.id() == pageId)
+                        return true;
+
+                    t = t.sibling;
+
+                    return t != null && t.page.id() == pageId;
+                }
 
-                if (t.fwd != null)
-                    t = t.fwd;
-                else
-                    t = t.down;
+                t = t.down;
             }
 
             return false;
@@ -2177,55 +2117,58 @@ public abstract class BPlusTree<L, T extends L> {
          * @param buf Buffer.
          * @param io IO.
          * @param lvl Level.
-         * @param back If the given page is back page for the current tail, otherwise it must be an upper level page.
+         * @param type Type.
          * @param idx Insertion index or negative flag describing if the page is primary in this tail branch.
          */
-        private void addTail(Page page, ByteBuffer buf, BPlusIO<L> io,
-            int lvl, boolean back, byte primary, int idx) {
-            assert primary == Tail.PRIMARY || primary == Tail.COMPLIMENTARY: primary;
+        private void addTail(Page page, ByteBuffer buf, BPlusIO<L> io, int lvl, byte type, int idx) {
+            Tail<L> t = new Tail<>(page, buf, io, type, lvl, idx);
 
-            Tail<L> t = new Tail<>(page, buf, io, primary == Tail.PRIMARY, lvl, idx);
+            if (tail == null)
+                tail = t;
+            else if (tail.lvl == lvl) { // Add on the same level.
+                assert tail.sibling == null; // Only two siblings on a single level.
 
-            if (back) {
-                assert tail != null;
-                assert tail.lvl == lvl : "must be on the same level as out forward";
+                if (type == Tail.EXACT) {
+                    assert tail.type != Tail.EXACT;
 
-                t.fwd = tail;
-            }
-            else {
-                assert tail == null || tail.lvl == lvl - 1: "must be on upper higher than current tail";
+                    if (tail.down != null) { // Take down from sibling, EXACT must own down link.
+                        t.down = tail.down;
+                        tail.down = null;
+                    }
+
+                    t.sibling = tail;
+                    tail = t;
+                }
+                else {
+                    assert tail.type == Tail.EXACT: tail.type;
 
-                // Grow the tail.
+                    tail.sibling = t;
+                }
+            }
+            else if (tail.lvl == lvl - 1) { // Add on top of existing level.
                 t.down = tail;
+                tail = t;
             }
-
-            tail = t;
+            else
+                throw new IllegalStateException();
         }
 
         /**
          * @param lvl Level.
-         * @param back Back page.
-         * @return Tail.
+         * @return Tail of {@link Tail#EXACT} type at the given level.
          */
-        private Tail<L> getTail(int lvl, boolean back) {
+        private Tail<L> getTail(int lvl) {
             assert tail != null;
             assert lvl >= 0 && lvl <= tail.lvl: lvl;
 
             Tail<L> t = tail;
 
-            while (t.lvl != lvl) {
-                if (t.fwd != null)
-                    t = t.fwd;
-                else
-                    t = t.down;
-
-                assert t != null: lvl;
-            }
+            while (t.lvl != lvl)
+                t = t.down;
 
-            if (back)
-                return t.fwd != null ? t : null;
+            assert t.type == Tail.EXACT: t.type; // All the down links must be of EXACT type.
 
-            return t.fwd != null ? t.fwd : t;
+            return t;
         }
     }
 
@@ -2234,10 +2177,13 @@ public abstract class BPlusTree<L, T extends L> {
      */
     private static class Tail<L> {
         /** */
-        static final byte PRIMARY = 1;
+        static final byte BACK = 0;
 
         /** */
-        static final byte COMPLIMENTARY = 2;
+        static final byte EXACT = 1;
+
+        /** */
+        static final byte FORWARD = 2;
 
         /** */
         final Page page;
@@ -2249,7 +2195,7 @@ public abstract class BPlusTree<L, T extends L> {
         final BPlusIO<L> io;
 
         /** */
-        final boolean primary;
+        byte type;
 
         /** */
         final byte lvl;
@@ -2257,21 +2203,22 @@ public abstract class BPlusTree<L, T extends L> {
         /** */
         final short idx;
 
-        /** */
-        Tail<L> fwd;
+        /** Only {@link #EXACT} tail can have either {@link #BACK} or {@link #FORWARD} sibling.*/
+        Tail<L> sibling;
 
-        /** */
+        /** Only {@link #EXACT} tail can point to {@link #EXACT} tail of lower level. */
         Tail<L> down;
 
         /**
          * @param page Write locked page.
          * @param buf Buffer.
          * @param io IO.
-         * @param primary Primary.
+         * @param type Type.
          * @param lvl Level.
          * @param idx Insertion index.
          */
-        private Tail(Page page, ByteBuffer buf, BPlusIO<L> io, boolean primary, int lvl, int idx) {
+        private Tail(Page page, ByteBuffer buf, BPlusIO<L> io, byte type, int lvl, int idx) {
+            assert type == BACK || type == EXACT || type == FORWARD: type;
             assert idx == Integer.MIN_VALUE || (idx >= 0 && idx <= Short.MAX_VALUE): idx ;
             assert lvl >= 0 && lvl <= Byte.MAX_VALUE: lvl;
             assert page != null;
@@ -2279,7 +2226,7 @@ public abstract class BPlusTree<L, T extends L> {
             this.page = page;
             this.buf = buf;
             this.io = io;
-            this.primary = primary;
+            this.type = type;
             this.lvl = (byte)lvl;
             this.idx = (short)idx;
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/939b883a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
index c8cc3b2..958fa6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
@@ -44,7 +44,7 @@ public abstract class BPlusInnerIO<L> extends BPlusIO<L> {
     }
 
     /** {@inheritDoc} */
-    @Override public final int getMaxCount(ByteBuffer buf) {
+    @Override public int getMaxCount(ByteBuffer buf) {
         // The structure of the page is the following:
         // |ITEMS_OFF|w|A|x|B|y|C|z|
         // where capital letters are data items, lowercase letters are 8 byte page references.

http://git-wip-us.apache.org/repos/asf/ignite/blob/939b883a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
index 9712ff0..3688a75 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
@@ -33,7 +33,7 @@ public abstract class BPlusLeafIO<L> extends BPlusIO<L> {
     }
 
     /** {@inheritDoc} */
-    @Override public final int getMaxCount(ByteBuffer buf) {
+    @Override public int getMaxCount(ByteBuffer buf) {
         return (buf.capacity() - ITEMS_OFF) / itemSize;
     }
 


[14/50] ignite git commit: ignite-db - minor

Posted by sb...@apache.org.
ignite-db - minor


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/29179ce9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/29179ce9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/29179ce9

Branch: refs/heads/ignite-db-x-10884
Commit: 29179ce9f3204652e21c5540222053fe844bcb9b
Parents: 939b883
Author: S.Vladykin <sv...@gridgain.com>
Authored: Sun Apr 24 19:26:55 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Sun Apr 24 19:26:55 2016 +0300

----------------------------------------------------------------------
 .../processors/cache/database/tree/BPlusTree.java | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/29179ce9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 58f07e7..05fa56b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -632,7 +632,7 @@ public abstract class BPlusTree<L, T extends L> {
     private void doFind(Get g) throws IgniteCheckedException {
         try {
             for (;;) { // Go down with retries.
-                g.initOperation();
+                g.init();
 
                 switch (findDown(g, g.rootId, 0L, g.rootLvl)) {
                     case Get.RETRY:
@@ -832,12 +832,12 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Removed row.
      * @throws IgniteCheckedException If failed.
      */
-    public final T doRemove(L row, boolean ceil) throws IgniteCheckedException {
+    private T doRemove(L row, boolean ceil) throws IgniteCheckedException {
         Remove r = new Remove(row, ceil);
 
         try {
             for (;;) {
-                r.initOperation();
+                r.init();
 
                 switch (removeDown(r, r.rootId, 0L, 0L, r.rootLvl)) {
                     case Remove.RETRY:
@@ -1018,7 +1018,7 @@ public abstract class BPlusTree<L, T extends L> {
 
         try {
             for (;;) { // Go down with retries.
-                p.initOperation();
+                p.init();
 
                 switch (putDown(p, p.rootId, 0L, p.rootLvl)) {
                     case Put.RETRY:
@@ -1325,13 +1325,13 @@ public abstract class BPlusTree<L, T extends L> {
         /** */
         long rmvId;
 
-        /** Starting point root level. May be outdated. Must be modified only in {@link Get#initOperation()}. */
+        /** Starting point root level. May be outdated. Must be modified only in {@link Get#init()}. */
         int rootLvl;
 
-        /** Starting point root ID. May be outdated. Must be modified only in {@link Get#initOperation()}. */
+        /** Starting point root ID. May be outdated. Must be modified only in {@link Get#init()}. */
         long rootId;
 
-        /** Meta page. Initialized by {@link Get#initOperation()}, released by {@link Get#releaseMeta()}. */
+        /** Meta page. Initialized by {@link Get#init()}, released by {@link Get#releaseMeta()}. */
         Page meta;
 
         /** */
@@ -1356,13 +1356,13 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
-         * Initialize the given operation.
+         * Initialize operation.
          *
          * !!! Symmetrically with this method must be called {@link Get#releaseMeta()} in {@code finally} block.
          *
          * @throws IgniteCheckedException If failed.
          */
-        final void initOperation() throws IgniteCheckedException {
+        final void init() throws IgniteCheckedException {
             if (meta == null)
                 meta = page(metaPageId);
 


[26/50] ignite git commit: ignite-db - wip

Posted by sb...@apache.org.
ignite-db - wip


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/892f1f51
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/892f1f51
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/892f1f51

Branch: refs/heads/ignite-db-x-10884
Commit: 892f1f51de8ad8a0051758d14d46624ceac403a6
Parents: 42ac3b3
Author: S.Vladykin <sv...@gridgain.com>
Authored: Tue Apr 26 07:12:08 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Tue Apr 26 07:12:08 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 130 +++++++++++++------
 .../processors/database/BPlusTreeSelfTest.java  |  16 +--
 2 files changed, 95 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/892f1f51/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 848d167..f0919d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -340,6 +340,9 @@ public abstract class BPlusTree<L, T extends L> {
             if (r.needReplaceInner == TRUE ||
                 // We need to make sure that we have back or forward to be able to merge.
                 ((r.fwdId != 0 || r.backId != 0) && mayMerge(cnt - 1, io.getMaxCount(buf)))) {
+                if (cnt == 1) // It was the last item.
+                    r.needMergeEmptyBranch = TRUE;
+
                 // If we have backId then we've already locked back page, nothing to do here.
                 if (r.fwdId != 0 && r.backId == 0)
                     r.lockForward(0);
@@ -960,6 +963,20 @@ public abstract class BPlusTree<L, T extends L> {
     }
 
     /**
+     * @param leftCnt Left count.
+     * @param rightCnt Right count.
+     * @param cap Capacity.
+     * @param leaf Is leaf pages.
+     * @return {@code true} If can merge these pages.
+     */
+    private boolean canMerge(int leftCnt, int rightCnt, int cap, boolean leaf) {
+        cap -= leftCnt;
+        cap -= rightCnt;
+
+        return cap > 0 || (cap == 0 && (leaf || leftCnt == 0 || rightCnt == 0));
+    }
+
+    /**
      * @param cnt Count.
      * @param cap Capacity.
      * @return {@code true} If may merge.
@@ -1614,6 +1631,9 @@ public abstract class BPlusTree<L, T extends L> {
         /** */
         byte needReplaceInner = FALSE;
 
+        /** */
+        byte needMergeEmptyBranch = FALSE;
+
         /** Removed row. */
         T removed;
 
@@ -1669,32 +1689,32 @@ public abstract class BPlusTree<L, T extends L> {
          * @throws IgniteCheckedException If failed.
          */
         private void mergeEmptyBranch() throws IgniteCheckedException {
+            assert needMergeEmptyBranch == TRUE;
+
             Tail<L> t = tail;
 
             assert t.getCount() > 0;
 
-            boolean leaf = false;
-
             // Find empty branch beginning.
             for (Tail<L> t0 = t.down; t0 != null; t0 = t0.down) {
                 assert t0.type == Tail.EXACT: t0.type;
 
                 if (t0.getCount() != 0)
                     t = t0;
-
-                leaf = t0.lvl == 0;
             }
 
-            if (!leaf) // Can not merge empty branches in the middle of tree.
-                return;
-
             while (t.lvl != 0) { // If we've found empty branch, merge it top down.
-                boolean res = merge(t, true);
+                boolean res = merge(t);
 
                 assert res;
 
+                if (needMergeEmptyBranch == TRUE)
+                    needMergeEmptyBranch = READY; // Need to mark that we've already done the first iteration.
+
                 t = t.down;
             }
+
+            assert t.lvl == 0: t.lvl;
         }
 
         /**
@@ -1703,10 +1723,15 @@ public abstract class BPlusTree<L, T extends L> {
          * @throws IgniteCheckedException
          */
         private boolean mergeDown(Tail<L> t) throws IgniteCheckedException {
-            if (t.down == null || t.down.sibling == null)
+            assert needMergeEmptyBranch == FALSE || needMergeEmptyBranch == DONE: needMergeEmptyBranch;
+
+            if (t.down == null)
                 return true;
 
-            return mergeDown(t.down) && merge(t, false);
+            if (t.down.sibling == null) // We've merged something there.
+                return false;
+
+            return mergeDown(t.down) && merge(t);
         }
 
         /**
@@ -1719,29 +1744,37 @@ public abstract class BPlusTree<L, T extends L> {
             assert !isFinished();
             assert tail.type == Tail.EXACT;
 
-            // Need to lock more.
-            if (tail.down == null || needReplaceInner == TRUE || tail.getCount() == 0)
+            // Need to lock more to be able to merge.
+            if (needReplaceInner == TRUE ||
+                (needMergeEmptyBranch == TRUE && (tail.down == null || tail.getCount() == 0)))
                 return false;
 
-            mergeEmptyBranch();
+            // Can merge only if tail is not a routing page.
+            if (tail.getCount() > 0) {
+                if (needMergeEmptyBranch == TRUE) {
+                    mergeEmptyBranch();
 
-            boolean mergeMore = mergeDown(tail);
+                    needMergeEmptyBranch = DONE;
+                }
 
-            if (needReplaceInner == READY) {
-                // Need to replace inner key with new max key for the left subtree.
-                replaceInner();
+                boolean mergeMore = mergeDown(tail);
 
-                needReplaceInner = DONE;
-            }
+                if (needReplaceInner == READY) {
+                    // Need to replace inner key with new max key for the left subtree.
+                    replaceInner(needMergeEmptyBranch == DONE);
 
-            if (tail.getCount() == 0 && tail.lvl != 0 && getRootLevel(meta) == tail.lvl)
-                freePage(tail.page, tail.buf, tail.io, tail.lvl, false); // Free root.
-            else if (mergeMore) {
-                doReleaseTail(tail.down);
+                    needReplaceInner = DONE;
+                }
 
-                tail.down = null;
+                if (tail.getCount() == 0 && tail.lvl != 0 && getRootLevel(meta) == tail.lvl)
+                    freePage(tail.page, tail.buf, tail.io, tail.lvl, false); // Free root.
+                else if (mergeMore) {
+                    doReleaseTail(tail.down);
 
-                return false;
+                    tail.down = null;
+
+                    return false;
+                }
             }
 
             releaseTail();
@@ -1868,11 +1901,10 @@ public abstract class BPlusTree<L, T extends L> {
          * @param prnt Parent tail.
          * @param left Left child tail.
          * @param right Right child tail.
-         * @param emptyBranch If we are merging an empty branch.
          * @return {@code true} If merged successfully.
          * @throws IgniteCheckedException If failed.
          */
-        private boolean doMerge(Tail<L> prnt, Tail<L> left, Tail<L> right, boolean emptyBranch)
+        private boolean doMerge(Tail<L> prnt, Tail<L> left, Tail<L> right)
             throws IgniteCheckedException {
             assert right.io == left.io; // Otherwise incompatible.
             assert left.io.getForward(left.buf) == right.page.id();
@@ -1885,6 +1917,8 @@ public abstract class BPlusTree<L, T extends L> {
 
             int newCnt = leftCnt + rightCnt;
 
+            boolean emptyBranch = needMergeEmptyBranch == TRUE || needMergeEmptyBranch == READY;
+
             // Need to move down split key in inner pages. For empty branch merge parent key will be just dropped.
             if (left.lvl != 0 && !emptyBranch)
                 newCnt++;
@@ -1911,15 +1945,14 @@ public abstract class BPlusTree<L, T extends L> {
                 leftCnt++;
             }
 
-            left.io.copyItems(right.buf, left.buf, 0, leftCnt, rightCnt, !emptyBranch || rightCnt != 0);
+            left.io.copyItems(right.buf, left.buf, 0, leftCnt, rightCnt, !emptyBranch);
             left.io.setForward(left.buf, right.io.getForward(right.buf));
 
             // Fix index for the right move: remove last item.
-            // For the empty branch we always remove the last item, because we always drop right child after merge.
             int prntIdx = prnt.idx == prntCnt ? prntCnt - 1 : prnt.idx;
 
-            // Remove split key from parent. Index -1 means that it is not actually our parent and we should not remove.
-            if (prntIdx != -1)
+            // Remove split key from parent. If we ar emerging empty branch then remove only on the top iteration.
+            if (needMergeEmptyBranch != READY)
                 doRemove(prnt.io, prnt.buf, prntCnt, prntIdx);
 
             // Forward page is now empty and has no links, can free and release it right away.
@@ -1972,26 +2005,34 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
+         * @param emptyBranchMerged If empty branch was merged.
          * @throws IgniteCheckedException If failed.
          */
-        private void replaceInner() throws IgniteCheckedException {
+        private void replaceInner(boolean emptyBranchMerged) throws IgniteCheckedException {
             assert needReplaceInner == READY: needReplaceInner;
-            assert tail.lvl > 0;
-            assert innerIdx >= 0;
+            assert tail.lvl > 0: "leaf";
+            assert innerIdx >= 0: innerIdx;
 
             Tail<L> leaf = getTail(0);
             Tail<L> inner = tail;
 
             assert inner.type == Tail.EXACT: inner.type;
-            assert innerIdx < inner.getCount();
 
-            int cnt = leaf.getCount();
+            int innerCnt = inner.getCount();
+            int leafCnt = leaf.getCount();
 
-            assert cnt > 0: cnt; // Leaf must be merged at this point already if it was empty.
+            assert leafCnt > 0: leafCnt; // Leaf must be merged at this point already if it was empty.
+
+            if (innerIdx == innerCnt) {
+                if (emptyBranchMerged)
+                    return; // Our inner key was dropped.
+
+                throw new IllegalStateException();
+            }
 
             if (innerIdx < inner.getCount()) {
                 // Update inner key with the new biggest key of left subtree.
-                inner.io.store(inner.buf, innerIdx, leaf.io, leaf.buf, cnt - 1);
+                inner.io.store(inner.buf, innerIdx, leaf.io, leaf.buf, leafCnt - 1);
                 leaf.io.setRemoveId(leaf.buf, globalRmvId.get());
             }
             else {
@@ -2003,11 +2044,10 @@ public abstract class BPlusTree<L, T extends L> {
 
         /**
          * @param prnt Parent for merge.
-         * @param emptyBranch If we are merging an empty branch.
          * @return {@code true} If merged, {@code false} if not (because of insufficient space or empty parent).
          * @throws IgniteCheckedException If failed.
          */
-        private boolean merge(Tail<L> prnt, boolean emptyBranch) throws IgniteCheckedException {
+        private boolean merge(Tail<L> prnt) throws IgniteCheckedException {
             if (prnt.getCount() == 0)
                 return false; // Parent is an empty routing page, child forward page will have another parent.
 
@@ -2026,7 +2066,7 @@ public abstract class BPlusTree<L, T extends L> {
 
             assert right.io == left.io: "must always be the same"; // Otherwise can be not compatible.
 
-            if (!doMerge(prnt, left, right, emptyBranch))
+            if (!doMerge(prnt, left, right))
                 return false;
 
             // left from BACK becomes EXACT.
@@ -2057,6 +2097,12 @@ public abstract class BPlusTree<L, T extends L> {
          * Release pages for all locked levels at the tail.
          */
         private void releaseTail() {
+//            U.dumpStack("releaseTail");
+//            X.println("------>");
+//            for (Tail<L> t = tail; t != null; t = t.down)
+//                X.println("" + t);
+//            X.println("------<");
+
             doReleaseTail(tail);
 
             tail = null;
@@ -2236,7 +2282,7 @@ public abstract class BPlusTree<L, T extends L> {
 
         /** {@inheritDoc} */
         @Override public String toString() {
-            return S.toString(Tail.class, this, "pageId", page.id(), "cnt", getCount());
+            return S.toString(Tail.class, this, "pageId", page.id(), "cnt", getCount(), "lvl", lvl, "sibling", sibling);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/892f1f51/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index de2ca10..54accf5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -27,16 +27,13 @@ import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.impl.PageMemoryImpl;
-import org.apache.ignite.internal.processors.cache.database.MetaStore;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
 import org.apache.ignite.internal.util.lang.GridCursor;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -95,12 +92,13 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         pageMem.start();
 
-        reuseList = new ReuseList(CACHE_ID, pageMem, 2, new MetaStore() {
-            @Override public IgniteBiTuple<FullPageId,Boolean> getOrAllocateForIndex(int cacheId, String idxName)
-                throws IgniteCheckedException {
-                return new T2<>(allocatePage(), true);
-            }
-        });
+        reuseList = null;
+//            new ReuseList(CACHE_ID, pageMem, 2, new MetaStore() {
+//            @Override public IgniteBiTuple<FullPageId,Boolean> getOrAllocateForIndex(int cacheId, String idxName)
+//                throws IgniteCheckedException {
+//                return new T2<>(allocatePage(), true);
+//            }
+//        });
     }
 
     /** {@inheritDoc} */


[20/50] ignite git commit: ignite-db - io copy fix

Posted by sb...@apache.org.
ignite-db - io copy fix


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3df530a9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3df530a9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3df530a9

Branch: refs/heads/ignite-db-x-10884
Commit: 3df530a9ae55c7d56d07c241f4fccde83ee97589
Parents: 96023b8
Author: S.Vladykin <sv...@gridgain.com>
Authored: Mon Apr 25 19:50:48 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Mon Apr 25 19:50:48 2016 +0300

----------------------------------------------------------------------
 .../cache/database/freelist/io/FreeLeafIO.java  | 13 +++++++++++
 .../cache/database/tree/io/BPlusIO.java         | 23 +++++++++++++++++---
 .../cache/database/tree/io/BPlusInnerIO.java    | 19 +++-------------
 .../cache/database/tree/io/BPlusLeafIO.java     |  7 +++---
 .../database/tree/reuse/io/ReuseLeafIO.java     |  8 +++++++
 .../query/h2/database/io/H2LeafIO.java          |  8 +++++++
 6 files changed, 56 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/3df530a9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
index 8503c7d..edde01c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
@@ -18,11 +18,13 @@
 package org.apache.ignite.internal.processors.cache.database.freelist.io;
 
 import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.processors.cache.database.freelist.FreeItem;
 import org.apache.ignite.internal.processors.cache.database.freelist.FreeTree;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
 
@@ -51,6 +53,17 @@ public class FreeLeafIO extends BPlusLeafIO<FreeItem> implements FreeIO {
     }
 
     /** {@inheritDoc} */
+    @Override public void store(ByteBuffer dst, int dstIdx, BPlusIO<FreeItem> srcIo, ByteBuffer src, int srcIdx)
+        throws IgniteCheckedException {
+        assert srcIo == this: srcIo;
+
+        int off = offset(dstIdx);
+
+        dst.putShort(off, getFreeSpace(src, srcIdx));
+        dst.putInt(off + 2, getPageIndex(src, srcIdx));
+    }
+
+    /** {@inheritDoc} */
     @Override public short getFreeSpace(ByteBuffer buf, int idx) {
         int off = offset(idx);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/3df530a9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java
index 33a8a69..fbffe21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java
@@ -161,8 +161,22 @@ public abstract class BPlusIO<L> extends PageIO {
      * @param buf Buffer.
      * @param idx Index.
      * @param row Lookup or full row.
+     * @throws IgniteCheckedException If failed.
+     */
+    public abstract void store(ByteBuffer buf, int idx, L row) throws IgniteCheckedException;
+
+    /**
+     * Store row info from the given source.
+     *
+     * @param dst Destination buffer
+     * @param dstIdx Destination index.
+     * @param srcIo Source IO.
+     * @param src Source buffer.
+     * @param srcIdx Source index.
+     * @throws IgniteCheckedException If failed.
      */
-    public abstract void store(ByteBuffer buf, int idx, L row);
+    public abstract void store(ByteBuffer dst, int dstIdx, BPlusIO<L> srcIo, ByteBuffer src, int srcIdx)
+        throws IgniteCheckedException;
 
     /**
      * Get lookup row.
@@ -176,7 +190,8 @@ public abstract class BPlusIO<L> extends PageIO {
     public abstract L getLookupRow(BPlusTree<L, ?> tree, ByteBuffer buf, int idx) throws IgniteCheckedException;
 
     /**
-     * Copy items from source buffer to destination buffer. Both pages must be of the same type.
+     * Copy items from source buffer to destination buffer.
+     * Both pages must be of the same type and the same version.
      *
      * @param src Source buffer.
      * @param dst Destination buffer.
@@ -184,6 +199,8 @@ public abstract class BPlusIO<L> extends PageIO {
      * @param dstIdx Destination begin index.
      * @param cnt Items count.
      * @param cpLeft Copy leftmost link (makes sense only for inner pages).
+     * @throws IgniteCheckedException If failed.
      */
-    public abstract void copyItems(ByteBuffer src, ByteBuffer dst, int srcIdx, int dstIdx, int cnt, boolean cpLeft);
+    public abstract void copyItems(ByteBuffer src, ByteBuffer dst, int srcIdx, int dstIdx, int cnt, boolean cpLeft)
+        throws IgniteCheckedException;
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/3df530a9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
index 958fa6c..60c8f58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
@@ -93,13 +93,13 @@ public abstract class BPlusInnerIO<L> extends BPlusIO<L> {
 
     /** {@inheritDoc} */
     @Override public final void copyItems(ByteBuffer src, ByteBuffer dst, int srcIdx, int dstIdx, int cnt,
-        boolean cpLeft) {
+        boolean cpLeft) throws IgniteCheckedException {
         assert srcIdx != dstIdx || src != dst;
 
         if (dstIdx > srcIdx) {
             for (int i = cnt - 1; i >= 0; i--) {
                 dst.putLong(offset(dstIdx + i, SHIFT_RIGHT), src.getLong(offset(srcIdx + i, SHIFT_RIGHT)));
-                dst.putLong(offset(dstIdx + i, SHIFT_LINK), src.getLong(offset(srcIdx + i, SHIFT_LINK)));
+                store(dst, dstIdx + i, this, src, srcIdx + i); // TODO optimize with copy by itemSize
             }
 
             if (cpLeft)
@@ -111,25 +111,12 @@ public abstract class BPlusInnerIO<L> extends BPlusIO<L> {
 
             for (int i = 0; i < cnt; i++) {
                 dst.putLong(offset(dstIdx + i, SHIFT_RIGHT), src.getLong(offset(srcIdx + i, SHIFT_RIGHT)));
-                dst.putLong(offset(dstIdx + i, SHIFT_LINK), src.getLong(offset(srcIdx + i, SHIFT_LINK)));
+                store(dst, dstIdx + i, this, src, srcIdx + i); // TODO optimize with copy by itemSize
             }
         }
     }
 
     /**
-     * Store row info from the given source.
-     *
-     * @param dst Destination buffer
-     * @param dstIdx Destination index.
-     * @param srcIo Source IO.
-     * @param src Source buffer.
-     * @param srcIdx Source index.
-     * @throws IgniteCheckedException If failed.
-     */
-    public abstract void store(ByteBuffer dst, int dstIdx, BPlusIO<L> srcIo, ByteBuffer src, int srcIdx)
-        throws IgniteCheckedException;
-
-    /**
      * @param idx Index of element.
      * @param shift It can be either link itself or left or right page ID.
      * @return Offset from byte buffer begin in bytes.

http://git-wip-us.apache.org/repos/asf/ignite/blob/3df530a9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
index 3688a75..1b86ef9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache.database.tree.io;
 
 import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
 
 /**
  * Abstract IO routines for B+Tree leaf pages.
@@ -39,16 +40,16 @@ public abstract class BPlusLeafIO<L> extends BPlusIO<L> {
 
     /** {@inheritDoc} */
     @Override public final void copyItems(ByteBuffer src, ByteBuffer dst, int srcIdx, int dstIdx, int cnt,
-        boolean cpLeft) {
+        boolean cpLeft) throws IgniteCheckedException {
         assert srcIdx != dstIdx || src != dst;
 
         if (dstIdx > srcIdx) {
             for (int i = cnt - 1; i >= 0; i--)
-                dst.putLong(offset(dstIdx + i), src.getLong(offset(srcIdx + i)));
+                store(dst, dstIdx + i, this, src, srcIdx + i); // TODO optimize with copy by itemSize
         }
         else {
             for (int i = 0; i < cnt; i++)
-                dst.putLong(offset(dstIdx + i), src.getLong(offset(srcIdx + i)));
+                store(dst, dstIdx + i, this, src, srcIdx + i); // TODO optimize with copy by itemSize
         }
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/3df530a9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
index 69d08c9..bf46024 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
 
@@ -45,6 +46,13 @@ public class ReuseLeafIO extends BPlusLeafIO<FullPageId> {
         buf.putLong(offset(idx), fullPageId.pageId());
     }
 
+    /** {@inheritDoc} */
+    @Override public void store(ByteBuffer dst, int dstIdx, BPlusIO<FullPageId> srcIo, ByteBuffer src, int srcIdx) {
+        assert srcIo == this;
+
+        dst.putLong(offset(dstIdx), getPageId(src, srcIdx));
+    }
+
     /**
      * @param buf Buffer.
      * @param idx Index.

http://git-wip-us.apache.org/repos/asf/ignite/blob/3df530a9/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
index b3d3ae0..c2c915a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.query.h2.database.io;
 import java.nio.ByteBuffer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
 import org.apache.ignite.internal.processors.query.h2.database.H2Tree;
@@ -52,6 +53,13 @@ public class H2LeafIO extends BPlusLeafIO<SearchRow> implements H2RowLinkIO {
     }
 
     /** {@inheritDoc} */
+    @Override public void store(ByteBuffer dst, int dstIdx, BPlusIO<SearchRow> srcIo, ByteBuffer src, int srcIdx) {
+        assert srcIo == this;
+
+        setLink(dst, dstIdx, getLink(src, srcIdx));
+    }
+
+    /** {@inheritDoc} */
     @Override public SearchRow getLookupRow(BPlusTree<SearchRow,?> tree, ByteBuffer buf, int idx)
         throws IgniteCheckedException {
         long link = getLink(buf, idx);


[28/50] ignite git commit: ignite-db - more fixes

Posted by sb...@apache.org.
ignite-db - more fixes


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/92e53151
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/92e53151
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/92e53151

Branch: refs/heads/ignite-db-x-10884
Commit: 92e53151f1e8d71aa950011aaa5a08317058fb67
Parents: a11f9f0
Author: S.Vladykin <sv...@gridgain.com>
Authored: Tue Apr 26 07:29:21 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Tue Apr 26 07:29:21 2016 +0300

----------------------------------------------------------------------
 .../processors/cache/database/tree/BPlusTree.java       | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/92e53151/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index eb91602..6173d72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -414,13 +414,15 @@ public abstract class BPlusTree<L, T extends L> {
             boolean found = idx >= 0;
 
             if (found) {
-                // We can not miss the inner value on down move because of `triangle` invariant, thus it must be TRUE.
-                assert r.needReplaceInner == TRUE: r.needReplaceInner;
-                assert idx <= Short.MAX_VALUE : idx;
+                if (io.canGetRow()) {
+                    // We can not miss the inner value on down move because of `triangle` invariant, thus it must be TRUE.
+                    assert r.needReplaceInner == TRUE : r.needReplaceInner;
+                    assert idx <= Short.MAX_VALUE : idx;
 
-                r.innerIdx = (short)idx;
+                    r.innerIdx = (short)idx;
 
-                r.needReplaceInner = READY;
+                    r.needReplaceInner = READY;
+                }
             }
             else {
                 idx = -idx - 1;


[41/50] ignite git commit: Metadata storage fixed.

Posted by sb...@apache.org.
Metadata storage fixed.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ca839c87
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ca839c87
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ca839c87

Branch: refs/heads/ignite-db-x-10884
Commit: ca839c87d5d0dbcbedcc2d65e6f34473bc9e5cfa
Parents: 956f083
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 26 22:08:59 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 26 22:08:59 2016 -0700

----------------------------------------------------------------------
 .../cache/database/MetadataStorage.java         | 20 +++++++++++++++-----
 .../database/MetadataStorageSelfTest.java       |  4 ++--
 2 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ca839c87/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
index 169c97c..5cb0ca1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.pagemem.Page;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 
 /**
@@ -100,12 +101,19 @@ public class MetadataStorage implements MetaStore {
                 try {
                     state.writePage = nextPageId;
 
-                    nextPageId = buf.getLong();
+                    buf = nextPage.getForRead();
 
-                    rootPageId = scanForIndexRoot(buf, idxNameBytes, cacheId);
+                    try {
+                        nextPageId = buf.getLong();
 
-                    if (rootPageId != null)
-                        return rootPageId;
+                        rootPageId = scanForIndexRoot(buf, idxNameBytes, cacheId);
+
+                        if (rootPageId != null)
+                            return rootPageId;
+                    }
+                    finally {
+                        nextPage.releaseRead();
+                    }
                 }
                 finally {
                     pageMem.releasePage(nextPage);
@@ -160,7 +168,7 @@ public class MetadataStorage implements MetaStore {
             try {
                 long nextPageId = writeBuf.getLong();
 
-                assert nextPageId == 0: nextPageId;
+                assert nextPageId == 0: "[nextPageId=" + U.hexLong(nextPageId) + ", writePageId=" + U.hexLong(writePageId) + ']';
 
                 // Position buffer to the last record.
                 writeBuf.position(state.position);
@@ -185,6 +193,8 @@ public class MetadataStorage implements MetaStore {
                     writePage = pageMem.page(newMeta);
                     writeBuf = writePage.getForWrite();
                     writePageId = newMeta.pageId();
+                    // Next page, just need to make an offset.
+                    writeBuf.putLong(0);
                 }
 
                 writeBuf.put((byte)idxNameBytes.length);

http://git-wip-us.apache.org/repos/asf/ignite/blob/ca839c87/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
index c25288c..e5e4879 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
@@ -36,8 +36,8 @@ import java.util.concurrent.ThreadLocalRandom;
  *
  */
 public class MetadataStorageSelfTest extends GridCommonAbstractTest {
-    /** */
-    public static final int PAGE_SIZE = 8 * 1024;
+    /** Make sure page is small enough to trigger multiple pages in a linked list. */
+    public static final int PAGE_SIZE = 1024;
 
     /** */
     private static File allocationPath;


[39/50] ignite git commit: ignite-db - minor

Posted by sb...@apache.org.
ignite-db - minor


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8bb76a80
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8bb76a80
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8bb76a80

Branch: refs/heads/ignite-db-x-10884
Commit: 8bb76a8065080bd9ba9230e51bbd32305f84e0c3
Parents: 83d7a5c
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 06:23:50 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 06:23:50 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/reuse/ReuseList.java     | 19 +++++++------------
 .../processors/database/BPlusTreeSelfTest.java   |  6 +++---
 2 files changed, 10 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/8bb76a80/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
index eb89da9..c1e3d53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
@@ -83,7 +83,7 @@ public final class ReuseList {
 
         assert tree != null;
 
-        // Avoid recursion on the same tree.
+        // Avoid recursion on the same tree to avoid dead lock.
         if (tree == client) {
             treeIdx++; // Go forward and take the next tree.
 
@@ -106,14 +106,8 @@ public final class ReuseList {
         if (trees == null)
             return null;
 
-        ReuseTree tree = tree(client);
-
         // Remove and return page at min possible position.
-        FullPageId res = tree.removeCeil(MIN, bag);
-
-        assert res != null || tree.size() == 0; // TODO remove
-
-        return res;
+        return tree(client).removeCeil(MIN, bag);
     }
 
     /**
@@ -125,15 +119,16 @@ public final class ReuseList {
         if (bag == null)
             return;
 
-        ReuseTree tree = tree(client);
-
-        for (;;) {
+        for (int i = client.randomInt(trees.length);;) {
             FullPageId pageId = bag.pollFreePage();
 
             if (pageId == null)
                 break;
 
-            tree.put(pageId, bag);
+            trees[i].put(pageId, bag);
+
+            if (++i == trees.length)
+                i = 0;
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/8bb76a80/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 7258f36..3d4a62e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -121,7 +121,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
         if (reuseList != null) {
             long size = reuseList.size();
 
-            assertTrue("Reuse size: " + size, size < CNT);
+            assertTrue("Reuse size: " + size, size < 2000);
         }
 
         pageMem.stop();
@@ -492,10 +492,10 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         Map<Long,Long> map = new HashMap<>();
 
-        for (int i = 0 ; i < 100_000; i++) {
+        for (int i = 0 ; i < 1_000_000; i++) {
             Long x = (long)tree.randomInt(CNT);
 
-            if (i % 1000 == 0)
+            if (i % 100_000 == 0)
                 X.println(" --> " + i + "  " + x);
 
             if (tree.randomInt(2) == 0)


[17/50] ignite git commit: ignite-db - wip

Posted by sb...@apache.org.
ignite-db - wip


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/96023b8d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/96023b8d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/96023b8d

Branch: refs/heads/ignite-db-x-10884
Commit: 96023b8de28085d03d6a0e24ef7efcd8b9f98bc6
Parents: f444783
Author: S.Vladykin <sv...@gridgain.com>
Authored: Mon Apr 25 04:09:21 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Mon Apr 25 04:09:21 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 71 +++++++++++++-------
 1 file changed, 45 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/96023b8d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 039fc3d..61e612a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -161,7 +161,9 @@ public abstract class BPlusTree<L, T extends L> {
     private final PageHandler<Get> search = new GetPageHandler<Get>() {
         @Override public int run0(Page page, ByteBuffer buf, BPlusIO<L> io, Get g, int lvl)
             throws IgniteCheckedException {
-            g.backId = 0; // Usually we'll go left down.
+            boolean needBackIfRouting = g.backId != 0;
+
+            g.backId = 0; // Usually we'll go left down and don't need it.
 
             int cnt = io.getCount(buf);
             int idx = findInsertionPoint(io, buf, cnt, g.row);
@@ -200,14 +202,16 @@ public abstract class BPlusTree<L, T extends L> {
             else {
                 assert idx == cnt;
                 // Here child's forward is unknown to us (we either go right or it is an empty "routing" page),
-                // need to ask our forward about the child's forward (it must be leftmost child or our forward page).
+                // need to ask our forward about the child's forward (it must be leftmost child of our forward page).
                 // This is ok from the locking standpoint because we take all locks in the forward direction.
                 long fwdId = io.getForward(buf);
 
-                g.fwdId = fwdId == 0 ? 0 : getLeftmostChild(fwdId);
+                g.fwdId = fwdId == 0 ? 0 : getChild(fwdId, false);
 
-                if (cnt != 0) // If empty, it is a routing page and we go to the left, otherwise we go to the right.
+                if (cnt != 0) // It is not a routing page and we are going to the right, can get backId here.
                     g.backId = inner(io).getLeft(buf, cnt - 1);
+                else if (needBackIfRouting && g.getClass() == Remove.class)
+                    return Remove.GO_DOWN_X; // Can't get backId here because of possible deadlock.
             }
 
             return Get.GO_DOWN;
@@ -426,9 +430,7 @@ public abstract class BPlusTree<L, T extends L> {
 
             // We don't have a back page, need to lock our forward and become a back for it.
             // If found then we are on inner replacement page, it will be a top parent, no need to lock forward.
-            boolean noBack = !found && r.fwdId != 0 && r.backId == 0;
-
-            if (noBack)
+            if (!found && r.fwdId != 0 && r.backId == 0)
                 r.lockForward(lvl);
 
             r.addTail(page, buf, io, lvl, Tail.EXACT, idx);
@@ -885,10 +887,16 @@ public abstract class BPlusTree<L, T extends L> {
                 // Init args.
                 r.pageId = pageId;
                 r.fwdId = fwdId;
+                r.backId = backId;
 
                 int res = readPage(page, search, r, lvl, Remove.RETRY);
 
                 switch (res) {
+                    case Remove.GO_DOWN_X:
+                        // We need to get backId here for our page, it must be the last child of our back.
+                        r.backId = getChild(backId, true);
+
+                        // Intentional fallthrough.
                     case Remove.GO_DOWN:
                         res = removeDown(r, r.pageId, r.backId, r.fwdId, lvl - 1);
 
@@ -929,7 +937,7 @@ public abstract class BPlusTree<L, T extends L> {
                         if (res == Remove.NOT_FOUND) {
                             assert r.ceil: "must be a retry if not a ceiling remove";
 
-                            r.finish(); // TODO may be try to remove from forward
+                            r.finish();
                         }
                         else if (res == Remove.FOUND && r.needReplaceInner == FALSE && r.needMerge == FALSE) {
                             // Finish if we don't need to do any merges.
@@ -1209,9 +1217,10 @@ public abstract class BPlusTree<L, T extends L> {
 
     /**
      * @param pageId Inner page ID.
-     * @return Leftmost child page ID.
+     * @param last If {@code true}, then get the last, else get the first child page.
+     * @return Child page ID.
      */
-    private long getLeftmostChild(long pageId) throws IgniteCheckedException {
+    private long getChild(long pageId, boolean last) throws IgniteCheckedException {
         try (Page page = page(pageId)) {
             assert page != null : "we've locked back page, forward can't be merged";
 
@@ -1220,9 +1229,11 @@ public abstract class BPlusTree<L, T extends L> {
             try {
                 BPlusIO<L> io = io(buf);
 
-                assert io.getCount(buf) >= 0; // Count can be 0 if it is a routing page.
+                // Count can be 0 here if it is a routing page, in this case we have single child.
+                int idx = last ? io.getCount(buf) : 0;
 
-                long res = inner(io).getLeft(buf, 0);
+                // getLeft(cnt) is the same as getRight(cnt - 1)
+                long res = inner(io).getLeft(buf, idx);
 
                 assert res != 0: "inner page with no route down: " + page.fullId();
 
@@ -1279,7 +1290,8 @@ public abstract class BPlusTree<L, T extends L> {
                         if (res == Put.RETRY_ROOT || p.isFinished())
                             return res;
 
-                        checkInterrupted();
+                        if (res == Put.RETRY)
+                            checkInterrupted();
 
                         continue; // We have to insert split row to this level or it is a retry.
 
@@ -1321,16 +1333,19 @@ public abstract class BPlusTree<L, T extends L> {
         static final int GO_DOWN = 1;
 
         /** */
-        static final int RETRY = 2;
+        static final int GO_DOWN_X = 2;
+
+        /** */
+        static final int FOUND = 5;
 
         /** */
-        static final int RETRY_ROOT = 3;
+        static final int NOT_FOUND = 6;
 
         /** */
-        static final int NOT_FOUND = 4;
+        static final int RETRY = 8;
 
         /** */
-        static final int FOUND = 5;
+        static final int RETRY_ROOT = 9;
 
         /** */
         long rmvId;
@@ -1682,14 +1697,18 @@ public abstract class BPlusTree<L, T extends L> {
             assert needMerge != FALSE || needReplaceInner != FALSE;
             assert tail != null;
 
+            if (needReplaceInner == TRUE)
+                return false; // Need to find inner page.
+
             if (needReplaceInner == READY) {
                 assert needMerge == FALSE: needMerge;
-                assert getTail(0) != null: "we must keep lock on the leaf page";
 
                 // We increment remove ID in write lock on leaf page, thus it is guaranteed that
                 // any successor will get greater value than he had read at the beginning of the operation.
-                // Thus it will be guaranteed to do a retry from root.
-                globalRmvId.incrementAndGet(); // TODO replace with pageId bit patching
+                // Thus he is guaranteed to do a retry from root. Since inner replace takes locks on the whole branch
+                // and releases the locks only when the inner key is updated and the successor saw the updated removeId,
+                // then after retry from root, he will see updated inner key.
+                globalRmvId.incrementAndGet();
 
                 // Need to replace inner key with new max key for the left subtree.
                 doReplaceInner();
@@ -1828,13 +1847,13 @@ public abstract class BPlusTree<L, T extends L> {
             throws IgniteCheckedException {
             assert cnt > 0;
             assert idx >= 0;
-            assert idx <= cnt;
-
-            if (idx == cnt) {
-                idx--; // This may happen in case of right turn, we need to remove the rightmost ref and link.
+            assert idx < cnt; // TODO check why is the code below exists
 
-                assert !kickLeftChild: "right child must be dropped here";
-            }
+//            if (idx == cnt) {
+//                idx--; // This may happen in case of right turn, we need to remove the rightmost ref and link.
+//
+//                assert !kickLeftChild: "right child must be dropped here";
+//            }
 
             cnt--;
 


[49/50] ignite git commit: ignite-db - cursor fix

Posted by sb...@apache.org.
ignite-db - cursor fix


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/06feabe9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/06feabe9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/06feabe9

Branch: refs/heads/ignite-db-x-10884
Commit: 06feabe9e3e133f71dcbb9e0cc5b3233cc21a8e2
Parents: 2895696
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 11:10:41 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 11:10:41 2016 +0300

----------------------------------------------------------------------
 .../internal/processors/cache/database/tree/BPlusTree.java       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/06feabe9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index c0144a7..cccb880 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -2566,7 +2566,9 @@ public abstract class BPlusTree<L, T extends L> {
                         int cmp = compare(io, buf, cnt - 1, upperBound);
 
                         if (cmp > 0) {
-                            cnt = findInsertionPoint(io, buf, cnt, upperBound) + 1;
+                            int idx = findInsertionPoint(io, buf, cnt, upperBound);
+
+                            cnt = idx < 0 ? -idx : idx + 1;
 
                             fwdId = 0; // The End.
                         }


[32/50] ignite git commit: Documentation for page ID.

Posted by sb...@apache.org.
Documentation for page ID.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a3581160
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a3581160
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a3581160

Branch: refs/heads/ignite-db-x-10884
Commit: a35811601e67fb025821d401515b41781b302a42
Parents: ad5aacf
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 26 10:57:37 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 26 10:57:37 2016 -0700

----------------------------------------------------------------------
 .../ignite/internal/pagemem/FullPageId.java     | 29 +++++++++++++++++---
 .../ignite/internal/pagemem/PageIdUtils.java    | 11 +++++---
 2 files changed, 32 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a3581160/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
index 0247aec..650a446 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
@@ -19,10 +19,31 @@ package org.apache.ignite.internal.pagemem;
 
 import org.apache.ignite.internal.util.typedef.internal.SB;
 
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
-
 /**
- *
+ * Compound object used to address a page in the global page space.
+ * <p>
+ * There are three types of pages in PageMemory system:
+ * <ul>
+ *     <li>DATA pages</li>
+ *     <li>INDEX pages</li>
+ *     <li>META pages</li>
+ * </ul>
+ * Generally, a full page ID consists of a cache ID and page ID. A page ID consists of
+ * file ID (22 bits) and page index (30 bits).
+ * Higher 12 bits of page ID are reserved for use in index pages to address entries inside data pages.
+ * File ID consists of 8 reserved bits, page type (2 bits) and partition ID (14 bits).
+ * Note that partition ID is not used in full page ID comparison for non-data pages.
+ * <p>
+ * The structure of a page ID is shown in the next diagram:
+ * <pre>
+ * +------------+--------+--+--------------+------------------------------+
+ * |   12 bits  | 8 bits |2b|   14 bits    |            30 bits           |
+ * +------------+--------+--+--------------+------------------------------+
+ * |   OFFSET   | RESRVD |FL| PARTITION ID |          PAGE INDEX          |
+ * +------------+--------+--+--------------+------------------------------+
+ *              |       FILE ID            |
+ *              +--------------------------+
+ * </pre>
  */
 public class FullPageId {
     /** */
@@ -42,7 +63,7 @@ public class FullPageId {
         this.pageId = PageIdUtils.pageId(pageId);
         this.cacheId = cacheId;
 
-        effectivePageId = PageIdUtils.flag(pageId) == FLAG_IDX ? PageIdUtils.effectiveIndexPageId(pageId) : this.pageId;
+        effectivePageId = PageIdUtils.effectivePageId(pageId);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/a3581160/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
index 3a14c5f..2fed4b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.pagemem;
 
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+
 /**
  * Utility class for page ID parts manipulation.
  *
@@ -54,7 +56,8 @@ public final class PageIdUtils {
     private static final int FLAG_MASK = ~(-1 << FLAG_SIZE);
 
     /** */
-    private static final long EFFECTIVE_INDEX_PAGE_ID_MASK = ((long)FLAG_MASK << (PAGE_IDX_SIZE + PART_ID_SIZE)) | PAGE_IDX_MASK;
+    private static final long EFFECTIVE_NON_DATA_PAGE_ID_MASK =
+        ((long)FLAG_MASK << (PAGE_IDX_SIZE + PART_ID_SIZE)) | PAGE_IDX_MASK;
 
     /** Maximum page number. */
     public static final int MAX_PAGE_NUM = (1 << PAGE_IDX_SIZE) - 1;
@@ -144,10 +147,10 @@ public final class PageIdUtils {
 
     /**
      * @param link Page link.
-     * @return Effective index page id.
+     * @return Effective page id.
      */
-    public static long effectiveIndexPageId(long link) {
-        return link & EFFECTIVE_INDEX_PAGE_ID_MASK;
+    public static long effectivePageId(long link) {
+        return flag(link) == FLAG_DATA ? pageId(link) : link & EFFECTIVE_NON_DATA_PAGE_ID_MASK;
     }
 
     /**


[48/50] ignite git commit: ignite-db - fixes

Posted by sb...@apache.org.
ignite-db - fixes


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/28956963
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/28956963
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/28956963

Branch: refs/heads/ignite-db-x-10884
Commit: 28956963fde508794993ef012503c153692c55e7
Parents: 788b6e8
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 11:02:31 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 11:02:31 2016 +0300

----------------------------------------------------------------------
 .../internal/processors/cache/database/freelist/FreeList.java | 7 ++++++-
 .../internal/processors/cache/database/freelist/FreeTree.java | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/28956963/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
index 721f0c0..4665b6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
@@ -75,6 +75,8 @@ public class FreeList {
     /** */
     private final PageHandler<FreeTree> removeRow = new PageHandler<FreeTree>() {
         @Override public int run(Page page, ByteBuffer buf, FreeTree tree, int itemId) throws IgniteCheckedException {
+            assert tree != null;
+
             DataPageIO io = DataPageIO.VERSIONS.forPage(buf);
 
             assert DataPageIO.check(itemId): itemId;
@@ -168,10 +170,13 @@ public class FreeList {
         assert link != 0;
 
         long pageId = PageIdUtils.pageId(link);
+        int partId = PageIdUtils.partId(pageId);
         int itemId = PageIdUtils.dwordsOffset(link);
 
+        FreeTree tree = tree(partId);
+
         try (Page page = pageMem.page(new FullPageId(pageId, cctx.cacheId()))) {
-            writePage(page, removeRow, null, itemId, -1);
+            writePage(page, removeRow, tree, itemId, -1);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/28956963/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
index c5138c5..17f047e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
@@ -91,7 +91,7 @@ public class FreeTree extends BPlusTree<FreeItem, FreeItem> {
         int res = Short.compare(((FreeIO)io).getFreeSpace(buf, idx), row.freeSpace());
 
         if (res == 0)
-            res = Long.compare(((FreeLeafIO)io).getPageIndex(buf, idx), FreeInnerIO.pageIndex(row.pageId()));
+            res = Integer.compare(((FreeIO)io).getPageIndex(buf, idx), FreeInnerIO.pageIndex(row.pageId()));
 
         return res;
     }


[25/50] ignite git commit: ignite-db - wip mm+

Posted by sb...@apache.org.
ignite-db - wip mm+


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/42ac3b3b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/42ac3b3b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/42ac3b3b

Branch: refs/heads/ignite-db-x-10884
Commit: 42ac3b3bdcd8b5d6fe00774c9cf950ab01c356ea
Parents: 74dfb8f
Author: S.Vladykin <sv...@gridgain.com>
Authored: Tue Apr 26 03:36:14 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Tue Apr 26 03:36:14 2016 +0300

----------------------------------------------------------------------
 .../ignite/internal/pagemem/impl/PageImpl.java  |   2 +-
 .../cache/database/tree/BPlusTree.java          | 148 +++++++++---
 .../processors/database/BPlusTreeSelfTest.java  | 240 +++++++------------
 3 files changed, 196 insertions(+), 194 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/42ac3b3b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
index 9f42d93..1e1fb78 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
@@ -243,7 +243,7 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
     boolean releaseReference() {
         int refs = refCntUpd.decrementAndGet(this);
 
-        assert refs >= 0;
+        assert refs >= 0: fullId.pageId();
 
         return refs == 0;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/42ac3b3b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index a24c38a..848d167 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandle
 import org.apache.ignite.internal.util.lang.GridCursor;
 import org.apache.ignite.internal.util.lang.GridTreePrinter;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler.readPage;
@@ -1254,8 +1255,8 @@ public abstract class BPlusTree<L, T extends L> {
                         assert p.pageId != pageId;
                         assert p.fwdId != fwdId || fwdId == 0;
 
-                        // TODO race on go down?
-                        if (p.needReplaceInner == TRUE) { // Need to replace ref in inner page.
+                        // Need to replace key in inner page. There is no race because we keep tail lock after split.
+                        if (p.needReplaceInner == TRUE) {
                             p.needReplaceInner = FALSE; // Protect from retries.
 
                             res = writePage(page, replace, p, lvl, Put.RETRY);
@@ -1665,6 +1666,50 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
+         * @throws IgniteCheckedException If failed.
+         */
+        private void mergeEmptyBranch() throws IgniteCheckedException {
+            Tail<L> t = tail;
+
+            assert t.getCount() > 0;
+
+            boolean leaf = false;
+
+            // Find empty branch beginning.
+            for (Tail<L> t0 = t.down; t0 != null; t0 = t0.down) {
+                assert t0.type == Tail.EXACT: t0.type;
+
+                if (t0.getCount() != 0)
+                    t = t0;
+
+                leaf = t0.lvl == 0;
+            }
+
+            if (!leaf) // Can not merge empty branches in the middle of tree.
+                return;
+
+            while (t.lvl != 0) { // If we've found empty branch, merge it top down.
+                boolean res = merge(t, true);
+
+                assert res;
+
+                t = t.down;
+            }
+        }
+
+        /**
+         * @param t Tail.
+         * @return {@code true} If merged successfully or end reached.
+         * @throws IgniteCheckedException
+         */
+        private boolean mergeDown(Tail<L> t) throws IgniteCheckedException {
+            if (t.down == null || t.down.sibling == null)
+                return true;
+
+            return mergeDown(t.down) && merge(t, false);
+        }
+
+        /**
          * Process tail and finish.
          *
          * @return {@code false} If failed to finish and we need to lock more pages up.
@@ -1675,12 +1720,12 @@ public abstract class BPlusTree<L, T extends L> {
             assert tail.type == Tail.EXACT;
 
             // Need to lock more.
-            if (tail.down == null || needReplaceInner == TRUE || tail.io.getCount(tail.buf) == 0)
+            if (tail.down == null || needReplaceInner == TRUE || tail.getCount() == 0)
                 return false;
 
-            // Merge from the top to the bottom.
-            for (Tail<L> t = tail; t.down != null; t = t.down)
-                merge(t);
+            mergeEmptyBranch();
+
+            boolean mergeMore = mergeDown(tail);
 
             if (needReplaceInner == READY) {
                 // Need to replace inner key with new max key for the left subtree.
@@ -1689,11 +1734,15 @@ public abstract class BPlusTree<L, T extends L> {
                 needReplaceInner = DONE;
             }
 
-//            if (cnt == 0 && lvl != 0 && getRootLevel(meta) == lvl) TODO
-//                freePage(page, buf, io, lvl); // Free root.
+            if (tail.getCount() == 0 && tail.lvl != 0 && getRootLevel(meta) == tail.lvl)
+                freePage(tail.page, tail.buf, tail.io, tail.lvl, false); // Free root.
+            else if (mergeMore) {
+                doReleaseTail(tail.down);
 
-//            if (needMergeMore)
-//                return false;
+                tail.down = null;
+
+                return false;
+            }
 
             releaseTail();
             finish();
@@ -1819,32 +1868,37 @@ public abstract class BPlusTree<L, T extends L> {
          * @param prnt Parent tail.
          * @param left Left child tail.
          * @param right Right child tail.
+         * @param emptyBranch If we are merging an empty branch.
          * @return {@code true} If merged successfully.
          * @throws IgniteCheckedException If failed.
          */
-        private boolean mergePages(Tail<L> prnt, Tail<L> left, Tail<L> right) throws IgniteCheckedException {
+        private boolean doMerge(Tail<L> prnt, Tail<L> left, Tail<L> right, boolean emptyBranch)
+            throws IgniteCheckedException {
             assert right.io == left.io; // Otherwise incompatible.
             assert left.io.getForward(left.buf) == right.page.id();
 
-            int prntCnt = prnt.io.getCount(prnt.buf);
-            int backCnt = left.io.getCount(left.buf);
-            int curCnt = right.io.getCount(right.buf);
+            int prntCnt = prnt.getCount();
+            int leftCnt = left.getCount();
+            int rightCnt = right.getCount();
 
             assert prntCnt > 0: prntCnt;
 
-            int newCnt = backCnt + curCnt;
+            int newCnt = leftCnt + rightCnt;
 
-            // Need to move down split key in inner pages, but for the last key it will be a special case merge.
-            if (left.lvl != 0 && false) // TODO
+            // Need to move down split key in inner pages. For empty branch merge parent key will be just dropped.
+            if (left.lvl != 0 && !emptyBranch)
                 newCnt++;
 
-            if (newCnt > left.io.getMaxCount(left.buf))
+            if (newCnt > left.io.getMaxCount(left.buf)) {
+                assert !emptyBranch;
+
                 return false;
+            }
 
             left.io.setCount(left.buf, newCnt);
 
             // Move down split key in inner pages.
-            if (left.lvl != 0) {
+            if (left.lvl != 0 && !emptyBranch) {
                 int prntIdx = prnt.idx;
 
                 if (prntIdx == prntCnt) // It was a right turn.
@@ -1852,22 +1906,24 @@ public abstract class BPlusTree<L, T extends L> {
 
                 // We can be sure that we have enough free space to store split key here,
                 // because we've done remove already and did not release child locks.
-                left.io.store(left.buf, backCnt, prnt.io, prnt.buf, prntIdx);
+                left.io.store(left.buf, leftCnt, prnt.io, prnt.buf, prntIdx);
 
-                backCnt++;
+                leftCnt++;
             }
 
-            left.io.copyItems(right.buf, left.buf, 0, backCnt, curCnt, true);
+            left.io.copyItems(right.buf, left.buf, 0, leftCnt, rightCnt, !emptyBranch || rightCnt != 0);
             left.io.setForward(left.buf, right.io.getForward(right.buf));
 
             // Fix index for the right move: remove last item.
+            // For the empty branch we always remove the last item, because we always drop right child after merge.
             int prntIdx = prnt.idx == prntCnt ? prntCnt - 1 : prnt.idx;
 
-            // Remove split key from parent.
-            doRemove(prnt.io, prnt.buf, prntCnt, prntIdx);
+            // Remove split key from parent. Index -1 means that it is not actually our parent and we should not remove.
+            if (prntIdx != -1)
+                doRemove(prnt.io, prnt.buf, prntCnt, prntIdx);
 
-            // Forward page is now empty and has no links.
-            freePage(right.page, right.buf, right.io, right.lvl);
+            // Forward page is now empty and has no links, can free and release it right away.
+            freePage(right.page, right.buf, right.io, right.lvl, true);
 
             return true;
         }
@@ -1877,9 +1933,10 @@ public abstract class BPlusTree<L, T extends L> {
          * @param buf Buffer.
          * @param io IO.
          * @param lvl Level.
+         * @param release Release write lock and release page.
          * @throws IgniteCheckedException If failed.
          */
-        private void freePage(Page page, ByteBuffer buf, BPlusIO io, int lvl)
+        private void freePage(Page page, ByteBuffer buf, BPlusIO io, int lvl, boolean release)
             throws IgniteCheckedException {
             if (getFirstPageId(meta, lvl) == page.id()) {
                 // This logic will handle root as well.
@@ -1900,7 +1957,8 @@ public abstract class BPlusTree<L, T extends L> {
 
             emptyPages.add(page.fullId());
 
-            writeUnlockAndClose(page);
+            if (release)
+                writeUnlockAndClose(page);
         }
 
         /**
@@ -1925,31 +1983,32 @@ public abstract class BPlusTree<L, T extends L> {
             Tail<L> inner = tail;
 
             assert inner.type == Tail.EXACT: inner.type;
-            assert innerIdx < inner.io.getCount(inner.buf);
+            assert innerIdx < inner.getCount();
 
-            int cnt = leaf.io.getCount(leaf.buf);
+            int cnt = leaf.getCount();
 
             assert cnt > 0: cnt; // Leaf must be merged at this point already if it was empty.
 
-            if (innerIdx < inner.io.getCount(inner.buf)) {
+            if (innerIdx < inner.getCount()) {
                 // Update inner key with the new biggest key of left subtree.
                 inner.io.store(inner.buf, innerIdx, leaf.io, leaf.buf, cnt - 1);
                 leaf.io.setRemoveId(leaf.buf, globalRmvId.get());
             }
             else {
                 // If after leaf merge parent have lost inner key, we don't need to update it anymore.
-                assert innerIdx == inner.io.getCount(inner.buf);
+                assert innerIdx == inner.getCount();
                 assert inner(inner.io).getLeft(inner.buf, innerIdx) == leaf.page.id();
             }
         }
 
         /**
          * @param prnt Parent for merge.
+         * @param emptyBranch If we are merging an empty branch.
          * @return {@code true} If merged, {@code false} if not (because of insufficient space or empty parent).
          * @throws IgniteCheckedException If failed.
          */
-        private boolean merge(Tail<L> prnt) throws IgniteCheckedException {
-            if (prnt.io.getCount(prnt.buf) == 0)
+        private boolean merge(Tail<L> prnt, boolean emptyBranch) throws IgniteCheckedException {
+            if (prnt.getCount() == 0)
                 return false; // Parent is an empty routing page, child forward page will have another parent.
 
             Tail<L> right = prnt.down;
@@ -1967,7 +2026,7 @@ public abstract class BPlusTree<L, T extends L> {
 
             assert right.io == left.io: "must always be the same"; // Otherwise can be not compatible.
 
-            if (!mergePages(prnt, left, right))
+            if (!doMerge(prnt, left, right, emptyBranch))
                 return false;
 
             // left from BACK becomes EXACT.
@@ -1998,10 +2057,15 @@ public abstract class BPlusTree<L, T extends L> {
          * Release pages for all locked levels at the tail.
          */
         private void releaseTail() {
-            Tail t = tail;
+            doReleaseTail(tail);
 
             tail = null;
+        }
 
+        /**
+         * @param t Tail.
+         */
+        private void doReleaseTail(Tail<L> t) {
             while (t != null) {
                 writeUnlockAndClose(t.page);
 
@@ -2162,6 +2226,18 @@ public abstract class BPlusTree<L, T extends L> {
             this.lvl = (byte)lvl;
             this.idx = (short)idx;
         }
+
+        /**
+         * @return Count.
+         */
+        private int getCount() {
+            return io.getCount(buf);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(Tail.class, this, "pageId", page.id(), "cnt", getCount());
+        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/42ac3b3b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 6dcb237..de2ca10 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -61,7 +61,10 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     private static final int CACHE_ID = 100500;
 
     /** */
-    private static int MAX_COUNT = 0;
+    private static int MAX_ITEMS_COUNT = 0;
+
+    /** */
+    private static int CNT = 10;
 
     /** */
     private static int PUT_INC = 1;
@@ -75,6 +78,11 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /** */
     private ReuseList reuseList;
 
+    @Override
+    protected long getTestTimeout() {
+        return 25 * 60 * 1000;
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         long seed = System.nanoTime();
@@ -99,233 +107,151 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     @Override protected void afterTest() throws Exception {
         pageMem.stop();
 
-        MAX_COUNT = 0;
+        MAX_ITEMS_COUNT = 0;
         PUT_INC = 1;
         RMV_INC = -1;
+        CNT = 10;
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testSingle1() throws IgniteCheckedException {
-        MAX_COUNT = 1;
+    public void testPutRemove_1_10_mm_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
         PUT_INC = -1;
         RMV_INC = -1;
 
-        doTestPutRemoveForwardBackward(true);
+        doTestPutRemove(true);
     }
 
     /**
-     * @param canGetRow Can get row from inner page.
      * @throws IgniteCheckedException If failed.
      */
-    private void doTestPutRemoveForwardBackward(boolean canGetRow) throws IgniteCheckedException {
-        TestTree tree = createTestTree(canGetRow);
-
-        long cnt = 10;
-
-        for (long x = PUT_INC > 0 ? 0 : cnt - 1; x >= 0 && x < cnt; x += PUT_INC) {
-            assertNull(tree.findOne(x));
-
-            tree.put(x);
-
-            assertEquals(x, tree.findOne(x).longValue());
-        }
-
-        X.println(tree.printTree());
-
-        assertNull(tree.findOne(-1L));
-
-        for (long x = 0; x < cnt; x++)
-            assertEquals(x, tree.findOne(x).longValue());
-
-        assertNull(tree.findOne(cnt));
-
-        for (long x = RMV_INC > 0 ? 0 : cnt - 1; x >= 0 && x < cnt; x += RMV_INC) {
-            X.println(" -- " + x);
-
-            assertEquals(x, tree.remove(x).longValue());
-
-            assertNull(tree.findOne(x));
-
-            X.println(tree.printTree());
-        }
+    public void testPutRemove_1_10_mm_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
+        PUT_INC = -1;
+        RMV_INC = -1;
 
-        assertFalse(tree.find(null, null).next());
+        doTestPutRemove(false);
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutFindRemoveForward0() throws IgniteCheckedException {
-        doTestPutFindRemoveForward(false);
-    }
+    public void testPutRemove_1_10_pm_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
+        PUT_INC = 1;
+        RMV_INC = -1;
 
-    /**
-     * @throws IgniteCheckedException If failed.
-     */
-    public void testPutFindRemoveForward1() throws IgniteCheckedException {
-        doTestPutFindRemoveForward(true);
+        doTestPutRemove(true);
     }
 
     /**
-     * @param canGetRow Can get row from inner page.
      * @throws IgniteCheckedException If failed.
      */
-    private void doTestPutFindRemoveForward(boolean canGetRow) throws IgniteCheckedException {
-        TestTree tree = createTestTree(canGetRow);
-
-        long cnt = 100_000;
-
-        for (long x = 0; x < cnt; x++) {
-            assertNull(tree.findOne(x));
-
-            tree.put(x);
-
-            assertEquals(x, tree.findOne(x).longValue());
-        }
-
-        assertNull(tree.findOne(-1L));
-
-        for (long x = 0; x < cnt; x++)
-            assertEquals(x, tree.findOne(x).longValue());
-
-        assertNull(tree.findOne(cnt));
-
-        for (long x = 0; x < cnt; x++) {
-            assertEquals(x, tree.remove(x).longValue());
-
-            assertNull(tree.findOne(x));
-        }
+    public void testPutRemove_1_10_pm_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
+        PUT_INC = 1;
+        RMV_INC = -1;
 
-        assertFalse(tree.find(null, null).next());
+        doTestPutRemove(false);
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemoveFindBackward0() throws IgniteCheckedException {
-        doTestPutFindRemoveBackward(false);
-    }
+    public void testPutRemove_1_10_pp_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
+        PUT_INC = 1;
+        RMV_INC = 1;
 
-    /**
-     * @throws IgniteCheckedException If failed.
-     */
-    public void testPutRemoveFindBackward1() throws IgniteCheckedException {
-        doTestPutFindRemoveBackward(true);
+        doTestPutRemove(true);
     }
 
     /**
-     * @param canGetRow Can get row from inner page.
      * @throws IgniteCheckedException If failed.
      */
-    private void doTestPutFindRemoveBackward(boolean canGetRow) throws IgniteCheckedException {
-        TestTree tree = createTestTree(canGetRow);
-
-        long cnt = 100_000;
-
-        for (long x = cnt - 1; x >= 0; x--) {
-            assertNull(tree.findOne(x));
-
-            tree.put(x);
-
-            assertEquals(x, tree.findOne(x).longValue());
-        }
-
-        assertNull(tree.findOne(cnt));
-
-        for (long x = cnt - 1; x >= 0; x--)
-            assertEquals(x, tree.findOne(x).longValue());
-
-        assertNull(tree.findOne(-1L));
-
-        for (long x = cnt - 1; x >= 0; x--) {
-            assertEquals(x, tree.remove(x).longValue());
-
-            assertNull(tree.findOne(x));
-        }
+    public void testPutRemove_1_10_pp_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
+        PUT_INC = 1;
+        RMV_INC = 1;
 
-        assertFalse(tree.find(null, null).next());
+        doTestPutRemove(false);
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemoveFindRandom0() throws IgniteCheckedException {
-        doTestPutRemoveFindRandom(false);
+    public void testPutRemove_1_10_mp_1() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
+        PUT_INC = -1;
+        RMV_INC = 1;
+
+        doTestPutRemove(true);
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemoveFindRandom1() throws IgniteCheckedException {
-        doTestPutRemoveFindRandom(true);
+    public void testPutRemove_1_10_mp_0() throws IgniteCheckedException {
+        MAX_ITEMS_COUNT = 1;
+        CNT = 10;
+        PUT_INC = -1;
+        RMV_INC = 1;
+
+        doTestPutRemove(false);
     }
 
     /**
      * @param canGetRow Can get row from inner page.
      * @throws IgniteCheckedException If failed.
      */
-    private void doTestPutRemoveFindRandom(boolean canGetRow) throws IgniteCheckedException {
+    private void doTestPutRemove(boolean canGetRow) throws IgniteCheckedException {
         TestTree tree = createTestTree(canGetRow);
 
-        Map<Long,Long> map = new HashMap<>();
-
-        int cnt = 100_000;
-
-        for (int i = 0, end = 30 * cnt; i < end; i++) {
-//            if (i % 1000 == 0)
-//                X.println(" -> " + i);
-
-            long x = tree.randomInt(cnt);
-
-            switch(tree.randomInt(3)) {
-                case 0:
-//                    X.println("Put: " + x);
-
-                    assertEquals(map.put(x, x), tree.put(x));
-
-                case 1:
-//                    X.println("Get: " + x);
-
-                    assertEquals(map.get(x), tree.findOne(x));
+        long cnt = 10;
 
-                    break;
+        for (long x = PUT_INC > 0 ? 0 : cnt - 1; x >= 0 && x < cnt; x += PUT_INC) {
+            assertNull(tree.findOne(x));
 
-                case 2:
-//                    X.println("Rmv: " + x);
+            tree.put(x);
 
-                    assertEquals(map.remove(x), tree.remove(x));
+            assertEquals(x, tree.findOne(x).longValue());
+        }
 
-                    break;
+        X.println(tree.printTree());
 
-                default:
-                    fail();
-            }
-        }
+        assertNull(tree.findOne(-1L));
 
-        assertFalse(map.isEmpty());
+        for (long x = 0; x < cnt; x++)
+            assertEquals(x, tree.findOne(x).longValue());
 
-        for (Long x : map.keySet())
-            assertEquals(map.get(x), tree.findOne(x));
+        assertNull(tree.findOne(cnt));
 
-        GridCursor<Long> cursor = tree.find(null, null);
+        for (long x = RMV_INC > 0 ? 0 : cnt - 1; x >= 0 && x < cnt; x += RMV_INC) {
+            X.println(" -- " + x);
 
-        while (cursor.next()) {
-            Long x = cursor.get();
+            assertEquals(x, tree.remove(x).longValue());
 
-            assert x != null;
+            X.println(tree.printTree());
 
-            assertEquals(map.remove(x), x);
+            assertNull(tree.findOne(x));
         }
 
-        assertTrue(map.isEmpty());
+        assertFalse(tree.find(null, null).next());
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testRandomRemove0() throws IgniteCheckedException {
+    public void _testRandomRemove0() throws IgniteCheckedException {
         // seed: 1461177795261173000, 1461187841179332000
         doTestRandomRemove(false);
     }
@@ -333,7 +259,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testRandomRemove1() throws IgniteCheckedException {
+    public void _testRandomRemove1() throws IgniteCheckedException {
         // seed: 1461188744119034000 1461188844311788000 1461189099834526000
         doTestRandomRemove(true);
     }
@@ -519,8 +445,8 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         /** {@inheritDoc} */
         @Override public int getMaxCount(ByteBuffer buf) {
-            if (MAX_COUNT != 0)
-                return MAX_COUNT;
+            if (MAX_ITEMS_COUNT != 0)
+                return MAX_ITEMS_COUNT;
 
             return super.getMaxCount(buf);
         }
@@ -558,8 +484,8 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         /** {@inheritDoc} */
         @Override public int getMaxCount(ByteBuffer buf) {
-            if (MAX_COUNT != 0)
-                return MAX_COUNT;
+            if (MAX_ITEMS_COUNT != 0)
+                return MAX_ITEMS_COUNT;
 
             return super.getMaxCount(buf);
         }


[10/50] ignite git commit: GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.

Posted by sb...@apache.org.
GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.

Fix.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/99915a6d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/99915a6d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/99915a6d

Branch: refs/heads/ignite-db-x-10884
Commit: 99915a6d8e22e01eff4244bc56b2be5be033bb7e
Parents: d2fedf6
Author: dkarachentsev <dk...@gridgain.com>
Authored: Fri Apr 22 16:51:10 2016 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Fri Apr 22 16:51:10 2016 +0300

----------------------------------------------------------------------
 .../internal/pagemem/impl/PageMemoryImpl.java   | 58 ++++++++++++++++++--
 1 file changed, 52 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/99915a6d/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
index 62a4e72..5ee8bf3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -944,11 +945,13 @@ public class  PageMemoryImpl implements PageMemory {
 
         final int cap = seg.loadedPages.capacity();
 
-        assert seg.loadedPages.size() >= RANDOM_PAGES_EVICT_NUM;
-
         if (seg.acquiredPages.size() >= seg.loadedPages.size())
             throw new OutOfMemoryException("No not acquired pages left for segment. Unable to evict.");
 
+        // With big number of random picked pages we may fall into infinite loop, because
+        // every time the same page may be found.
+        SimpleLongSet ignored = null;
+
         while (true) {
             for (int i = 0; i < RANDOM_PAGES_EVICT_NUM; i++) {
                 // We need to lookup for pages only in current segment for thread safety,
@@ -958,7 +961,10 @@ public class  PageMemoryImpl implements PageMemory {
 
                 assert addr != INVALID_REL_PTR;
 
-                pageRelAddrs[i] = addr;
+                if (ignored != null && ignored.contains(addr))
+                    i--;
+                else
+                    pageRelAddrs[i] = addr;
             }
 
             final long relEvictAddr = findSuitablePageRelAddr(pageRelAddrs);
@@ -967,16 +973,31 @@ public class  PageMemoryImpl implements PageMemory {
 
             final long absEvictAddr = absolute(relEvictAddr);
 
-            assert absEvictAddr != dbMetaPageIdPtr;
-
             final FullPageId fullPageId = readFullPageId(absEvictAddr);
 
+            final long metaPageId = mem.readLong(dbMetaPageIdPtr);
+
+            if (fullPageId.pageId() == metaPageId && fullPageId.cacheId() == 0) {
+                if (ignored == null)
+                    ignored = new SimpleLongSet();
+
+                ignored.add(relEvictAddr);
+
+                continue;
+            }
+
             assert seg.writeLock().isHeldByCurrentThread();
 
             if (!seg.acquiredPages.containsKey(fullPageId))
                 seg.loadedPages.remove(fullPageId);
-            else
+            else {
+                if (ignored == null)
+                    ignored = new SimpleLongSet();
+
+                ignored.add(relEvictAddr);
+
                 continue;
+            }
 
             // Force flush data and free page.
             if (isDirty(absEvictAddr)) {
@@ -1039,6 +1060,31 @@ public class  PageMemoryImpl implements PageMemory {
     }
 
     /**
+     * Long set based on BitSet.
+     */
+    private static class SimpleLongSet {
+        /** */
+        final BitSet bitSet = new BitSet();
+
+        /**
+         * Add value to set.
+         *
+         * @param val Value.
+         */
+        void add(final long val) {
+            bitSet.set((int)(val % Integer.MAX_VALUE));
+        }
+
+        /**
+         * @param val Value to check.
+         * @return {@code True} if set contains value.
+         */
+        boolean contains(final long val) {
+            return bitSet.get((int)(val % Integer.MAX_VALUE));
+        }
+    }
+
+    /**
      *
      */
     private class Segment extends ReentrantReadWriteLock {


[08/50] ignite git commit: GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.

Posted by sb...@apache.org.
GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d2fedf6b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d2fedf6b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d2fedf6b

Branch: refs/heads/ignite-db-x-10884
Commit: d2fedf6bb95bdd5f30a800959f9c9ff93d200bc7
Parents: 4282d80
Author: dkarachentsev <dk...@gridgain.com>
Authored: Thu Apr 21 19:15:04 2016 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Thu Apr 21 19:15:04 2016 +0300

----------------------------------------------------------------------
 .../ignite/internal/pagemem/PageMemory.java     |   3 +-
 .../internal/pagemem/impl/FullPageIdTable.java  |  90 +++++++-
 .../ignite/internal/pagemem/impl/PageImpl.java  |   9 +
 .../internal/pagemem/impl/PageMemoryImpl.java   | 206 +++++++++++++++++--
 4 files changed, 287 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/d2fedf6b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
index 0f10002..cf5ff69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
@@ -73,7 +73,8 @@ public interface PageMemory extends LifecycleAware, PageIdAllocator {
      * @param pageId Page ID to get byte buffer for. The page ID must be present in the collection returned by
      *      the {@link #beginCheckpoint()} method call.
      * @param tmpBuf Temporary buffer to write changes into.
+     * @return {@code True} if data were read, {@code false} otherwise (data already saved to storage).
      * @throws IgniteException If failed to obtain page data.
      */
-    public void getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf);
+    public boolean getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf);
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/d2fedf6b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
index e05d64c..c6529f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
@@ -78,6 +78,24 @@ public class FullPageIdTable {
     /** */
     protected DirectMemoryUtils mem;
 
+    /** Addressing strategy. */
+    private final AddressingStrategy strategy;
+
+    /** Specifies types of addressing. */
+    public enum AddressingStrategy {
+        /**
+         * Insertion will search for each available cell.
+         * Slower, but more suitable when used many removes/insertions.
+         */
+        LINEAR,
+
+        /**
+         * Insertion will search for available cell with limited steps.
+         * Faster, but requires more memory to resolve collisions.
+         */
+        QUADRATIC
+    }
+
     /**
      * @return Estimated memory size required for this map to store the given number of elements.
      */
@@ -93,11 +111,20 @@ public class FullPageIdTable {
      * @param len Allocated memory length.
      * @param clear If {@code true}, then memory is considered dirty and will be cleared. Otherwise,
      *      map will assume that the given memory region is in valid state.
+     * @param stgy Addressing strategy {@link AddressingStrategy}.
      */
-    public FullPageIdTable(DirectMemoryUtils mem, long addr, long len, boolean clear) {
+    public FullPageIdTable(DirectMemoryUtils mem, long addr, long len, boolean clear, AddressingStrategy stgy) {
         valPtr = addr;
+        this.strategy = stgy;
         capacity = (int)((len - 4) / BYTES_PER_ENTRY);
-        maxSteps = (int)Math.sqrt(capacity);
+
+        if (stgy == AddressingStrategy.LINEAR)
+            maxSteps = capacity;
+        else if (stgy == AddressingStrategy.QUADRATIC)
+            maxSteps = (int) Math.sqrt(capacity);
+        else
+            throw new IllegalArgumentException("Unsupported addressing strategy: " + stgy);
+
         this.mem = mem;
 
         if (clear)
@@ -167,6 +194,24 @@ public class FullPageIdTable {
     }
 
     /**
+     * Find nearest value from specified position to the right.
+     *
+     * @param idx Index to start searching from.
+     * @param absent Default value that will be returned if no values present.
+     * @return Closest value to the index or {@code absent} if no values found.
+     */
+    public long getNearestAt(final int idx, final long absent) {
+        for (int i = idx; i < capacity + idx; i++) {
+            final int idx2 = i >= capacity ? i - capacity : i;
+
+            if (isValuePresentAt(idx2))
+                return valueAt(idx2);
+        }
+
+        return absent;
+    }
+
+    /**
      * @param key Key.
      * @return Key index.
      */
@@ -178,7 +223,7 @@ public class FullPageIdTable {
         do {
             int res = testKeyAt(index, key);
 
-            if (res == EMPTY) {
+            if (res == EMPTY || res == REMOVED) {
                 setKeyAt(index, key);
 
                 incrementSize();
@@ -188,9 +233,14 @@ public class FullPageIdTable {
             else if (res == EQUAL)
                 return index;
             else
-                assert res == REMOVED || res == NOT_EQUAL;
+                assert res == NOT_EQUAL;
+
+            if (strategy == AddressingStrategy.QUADRATIC)
+                index += step;
+            else if (strategy == AddressingStrategy.LINEAR)
+                index++;
 
-            if ((index += step) >= capacity)
+            if (index >= capacity)
                 index -= capacity;
         }
         while (++step <= maxSteps);
@@ -217,7 +267,12 @@ public class FullPageIdTable {
             else
                 assert res == REMOVED || res == NOT_EQUAL;
 
-            if ((index += step) >= capacity)
+            if (strategy == AddressingStrategy.QUADRATIC)
+                index += step;
+            else if (strategy == AddressingStrategy.LINEAR)
+                index++;
+
+            if (index >= capacity)
                 index -= capacity;
         } while (++step <= maxSteps);
 
@@ -248,7 +303,12 @@ public class FullPageIdTable {
             else
                 assert res == REMOVED || res == NOT_EQUAL;
 
-            if ((index += step) >= capacity)
+            if (strategy == AddressingStrategy.QUADRATIC)
+                index += step;
+            else if (strategy == AddressingStrategy.LINEAR)
+                index++;
+
+            if (index >= capacity)
                 index -= capacity;
         }
         while (++step <= maxSteps);
@@ -278,6 +338,20 @@ public class FullPageIdTable {
     }
 
     /**
+     * @param idx Index to test.
+     * @return {@code True} if value set for index.
+     */
+    private boolean isValuePresentAt(final int idx) {
+        long base = valPtr + 4 + (long)idx * BYTES_PER_ENTRY;
+
+        long pageId = mem.readLong(base);
+        int cacheId = mem.readInt(base + 8);
+
+        return !((pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID)
+            || (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID));
+    }
+
+    /**
      * @param fullId Full page ID to check.
      * @return {@code True} if checks succeeded.
      */
@@ -333,6 +407,6 @@ public class FullPageIdTable {
      *
      */
     private void decrementSize() {
-        mem.writeInt(valPtr, mem.readInt(valPtr) + 1);
+        mem.writeInt(valPtr, mem.readInt(valPtr) - 1);
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/d2fedf6b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
index 77f096d..6e33572 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
@@ -141,6 +141,8 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
     @Override public ByteBuffer getForRead() {
         acquireShared(1);
 
+        pageMem.atomicWriteCurrentTimestamp(ptr);
+
         return reset(buf.asReadOnlyBuffer());
     }
 
@@ -156,6 +158,8 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
 
         markDirty();
 
+        pageMem.atomicWriteCurrentTimestamp(ptr);
+
         return reset(buf);
     }
 
@@ -167,6 +171,8 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
             acquire(1);
 
             setExclusiveOwnerThread(th);
+
+            pageMem.atomicWriteCurrentTimestamp(ptr);
         }
 
         return reset(buf);
@@ -211,6 +217,7 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
         return pageMem.isDirty(ptr);
     }
 
+    /** {@inheritDoc} */
     @Override public String toString() {
         SB sb = new SB("PageImpl [handle=");
 
@@ -230,6 +237,8 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
      */
     void acquireReference() {
         refCntUpd.incrementAndGet(this);
+
+        pageMem.atomicWriteCurrentTimestamp(ptr);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/d2fedf6b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
index b8d58f6..62a4e72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
@@ -26,6 +26,7 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.ignite.IgniteCheckedException;
@@ -79,8 +80,20 @@ public class  PageMemoryImpl implements PageMemory {
     /** Page ID offset  */
     public static final int PAGE_ID_OFFSET = 16;
 
-    /** Need a 8-byte pointer for linked list and 8 bytes for internal needs. */
-    public static final int PAGE_OVERHEAD = 24;
+    /** Page cache ID offset. */
+    public static final int PAGE_CACHE_ID_OFFSET = 24;
+
+    /** Page access timestamp */
+    public static final int PAGE_TIMESTAMP_OFFSET = 28;
+
+    /**
+     * Need a 8-byte pointer for linked list, 8 bytes for internal needs (flags),
+     * 4 bytes cache ID, 8 bytes timestamp.
+     */
+    public static final int PAGE_OVERHEAD = 36;
+
+    /** Number of random pages that will be picked for eviction. */
+    public static final int RANDOM_PAGES_EVICT_NUM = 5;
 
     /** Page size. */
     private int sysPageSize;
@@ -210,9 +223,14 @@ public class  PageMemoryImpl implements PageMemory {
 
             relPtr = allocateFreePage();
 
+            if (relPtr == INVALID_REL_PTR)
+                throw new OutOfMemoryException();
+
             absPtr = absolute(relPtr);
 
             writePageId(absPtr, pageId);
+            writePageCacheId(absPtr, cacheId);
+            atomicWriteCurrentTimestamp(absPtr);
         }
 
         // TODO pass an argument to decide whether the page should be cleaned.
@@ -309,8 +327,14 @@ public class  PageMemoryImpl implements PageMemory {
 
                 relPtr = borrowOrAllocateFreePage();
 
+                if (relPtr == INVALID_REL_PTR)
+                    relPtr = evictPage(seg);
+
                 long absPtr = absolute(relPtr);
 
+                writeFullPageId(absPtr, fullId);
+                atomicWriteCurrentTimestamp(absPtr);
+
                 // We can clear dirty flag after the page has been allocated.
                 setDirty(fullId, absPtr, false);
 
@@ -373,7 +397,7 @@ public class  PageMemoryImpl implements PageMemory {
     }
 
     /** {@inheritDoc} */
-    @Override public void getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf) {
+    @Override public boolean getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf) {
         assert tmpBuf.remaining() == pageSize();
 
         Segment seg = segment(pageId);
@@ -386,15 +410,16 @@ public class  PageMemoryImpl implements PageMemory {
             page = seg.acquiredPages.get(pageId);
 
             if (page != null) {
-                assert page.isDirty() : "Page is acquired for a checkpoint, but is not dirty: " + page;
+                if (!page.isDirty())
+                    return false;
 
                 page.acquireReference();
             }
             else {
                 long relPtr = seg.loadedPages.get(pageId, INVALID_REL_PTR);
 
-                assert relPtr != INVALID_REL_PTR : "Failed to get page checkpoint data (page has been evicted) " +
-                    "[pageId=" + pageId + ']';
+                if (relPtr == INVALID_REL_PTR)
+                    return false;
 
                 long absPtr = absolute(relPtr);
 
@@ -404,7 +429,7 @@ public class  PageMemoryImpl implements PageMemory {
 
                 setDirty(pageId, absPtr, false);
 
-                return;
+                return true;
             }
         }
         finally {
@@ -428,6 +453,8 @@ public class  PageMemoryImpl implements PageMemory {
         finally {
             releasePage(page);
         }
+
+        return true;
     }
 
     /**
@@ -529,6 +556,47 @@ public class  PageMemoryImpl implements PageMemory {
     }
 
     /**
+     * Reads cache ID from the page at the given absolute pointer.
+     *
+     * @param absPtr Absolute memory pointer to the page header.
+     * @return Cache ID written to the page.
+     */
+    int readPageCacheId(final long absPtr) {
+        return mem.readInt(absPtr + PAGE_CACHE_ID_OFFSET);
+    }
+
+    /**
+     * Writes cache ID from the page at the given absolute pointer.
+     *
+     * @param absPtr Absolute memory pointer to the page header.
+     * @param cacheId Cache ID to write.
+     */
+    void writePageCacheId(final long absPtr, final int cacheId) {
+        mem.writeInt(absPtr + PAGE_CACHE_ID_OFFSET, cacheId);
+    }
+
+    /**
+     * Reads page ID and cache ID from the page at the given absolute pointer.
+     *
+     * @param absPtr Absolute memory pointer to the page header.
+     * @return Full page ID written to the page.
+     */
+    FullPageId readFullPageId(final long absPtr) {
+        return new FullPageId(readPageId(absPtr), readPageCacheId(absPtr));
+    }
+
+    /**
+     * Writes page ID and cache ID from the page at the given absolute pointer.
+     *
+     * @param absPtr Absolute memory pointer to the page header.
+     * @param fullPageId Full page ID to write.
+     */
+    void writeFullPageId(final long absPtr, final FullPageId fullPageId) {
+        writePageId(absPtr, fullPageId.pageId());
+        writePageCacheId(absPtr, fullPageId.cacheId());
+    }
+
+    /**
      * @param absPtr Absolute pointer.
      * @return {@code True} if page is dirty.
      */
@@ -576,6 +644,23 @@ public class  PageMemoryImpl implements PageMemory {
             dirtyPages.remove(pageId);
     }
 
+    void atomicWriteCurrentTimestamp(final long absPtr) {
+        while (true) {
+            final long readTs = readTimestamp(absPtr);
+
+            if (mem.compareAndSwapLong(absPtr + PAGE_TIMESTAMP_OFFSET, readTs, U.currentTimeMillis()))
+                break;
+        }
+    }
+
+    void writeTimestamp(final long absPtr, final long ts) {
+        mem.writeLong(absPtr + PAGE_TIMESTAMP_OFFSET, ts);
+    }
+
+    long readTimestamp(final long absPtr) {
+        return mem.readLong(absPtr + PAGE_TIMESTAMP_OFFSET);
+    }
+
     /**
      * Attempts to restore page memory state based on the memory chunks returned by the allocator.
      */
@@ -749,14 +834,14 @@ public class  PageMemoryImpl implements PageMemory {
     /**
      * Requests next memory chunk from the system allocator.
      */
-    private void requestNextChunk() {
+    private boolean requestNextChunk() {
         assert Thread.holdsLock(this);
 
         int curIdx = currentChunk.idx;
 
         // If current chunk is the last one, fail.
         if (curIdx == chunks.size() - 1)
-            throw new OutOfMemoryException();
+            return false;
 
         Chunk chunk = chunks.get(curIdx + 1);
 
@@ -765,6 +850,8 @@ public class  PageMemoryImpl implements PageMemory {
                 ", base=0x" + U.hexLong(chunk.fr.address()) + ", len=" + chunk.size() + ']');
 
         currentChunk = chunk;
+
+        return true;
     }
 
     /**
@@ -834,8 +921,8 @@ public class  PageMemoryImpl implements PageMemory {
                 synchronized (this) {
                     Chunk full = currentChunk;
 
-                    if (chunk == full)
-                        requestNextChunk();
+                    if (chunk == full && !requestNextChunk())
+                        return INVALID_REL_PTR;
                 }
             }
             else
@@ -844,6 +931,98 @@ public class  PageMemoryImpl implements PageMemory {
     }
 
     /**
+     * Evict random oldest page from memory to storage.
+     *
+     * @param seg Currently locked segment.
+     * @return Relative addres for evicted page.
+     * @throws IgniteCheckedException
+     */
+    private long evictPage(final Segment seg) throws IgniteCheckedException {
+        final ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        final long[] pageRelAddrs = new long[RANDOM_PAGES_EVICT_NUM];
+
+        final int cap = seg.loadedPages.capacity();
+
+        assert seg.loadedPages.size() >= RANDOM_PAGES_EVICT_NUM;
+
+        if (seg.acquiredPages.size() >= seg.loadedPages.size())
+            throw new OutOfMemoryException("No not acquired pages left for segment. Unable to evict.");
+
+        while (true) {
+            for (int i = 0; i < RANDOM_PAGES_EVICT_NUM; i++) {
+                // We need to lookup for pages only in current segment for thread safety,
+                // so peeking random memory will lead to checking for found page segment.
+                // It's much faster to check available pages for segment right away.
+                final long addr = seg.loadedPages.getNearestAt(rnd.nextInt(cap), INVALID_REL_PTR);
+
+                assert addr != INVALID_REL_PTR;
+
+                pageRelAddrs[i] = addr;
+            }
+
+            final long relEvictAddr = findSuitablePageRelAddr(pageRelAddrs);
+
+            assert relEvictAddr != INVALID_REL_PTR;
+
+            final long absEvictAddr = absolute(relEvictAddr);
+
+            assert absEvictAddr != dbMetaPageIdPtr;
+
+            final FullPageId fullPageId = readFullPageId(absEvictAddr);
+
+            assert seg.writeLock().isHeldByCurrentThread();
+
+            if (!seg.acquiredPages.containsKey(fullPageId))
+                seg.loadedPages.remove(fullPageId);
+            else
+                continue;
+
+            // Force flush data and free page.
+            if (isDirty(absEvictAddr)) {
+                storeMgr.write(fullPageId.cacheId(), fullPageId.pageId(), wrapPointer(absEvictAddr + PAGE_OVERHEAD, pageSize()));
+
+                setDirty(fullPageId, absEvictAddr, false);
+            }
+
+            return relEvictAddr;
+        }
+    }
+
+    /**
+     * Find oldest and preferable not dirty page from passed ones.
+     *
+     * @param relAddrs Addresses to find from.
+     * @return The oldest and may be dirty page relative address.
+     */
+    private long findSuitablePageRelAddr(final long[] relAddrs) {
+        long addr = INVALID_REL_PTR;
+        long ts = Long.MAX_VALUE;
+        long dirtyTs = Long.MAX_VALUE;
+        long dirtyAddr = INVALID_REL_PTR;
+
+        for (final long relAddr : relAddrs) {
+            final long absPageAddr = absolute(relAddr);
+
+            final long pageTs = readTimestamp(absPageAddr);
+
+            final boolean dirty = isDirty(absPageAddr);
+
+            if (pageTs < ts && !dirty) {
+                addr = relAddr;
+
+                ts = pageTs;
+            } else if (pageTs < dirtyTs && dirty) {
+                dirtyAddr = relAddr;
+
+                dirtyTs = pageTs;
+            }
+        }
+
+        return addr == INVALID_REL_PTR ? dirtyAddr : addr;
+    }
+
+    /**
      * @param relPtr Relative pointer to free.
      */
     private void releaseFreePage(long relPtr) {
@@ -877,7 +1056,10 @@ public class  PageMemoryImpl implements PageMemory {
          * @param len Length of the allocated memory.
          */
         private Segment(long ptr, long len, boolean clear) {
-            loadedPages = new FullPageIdTable(mem, ptr, len, clear);
+            loadedPages = new FullPageIdTable(mem, ptr, len, clear,
+                storeMgr == null // if null evictions won't be used
+                    ? FullPageIdTable.AddressingStrategy.QUADRATIC
+                    : FullPageIdTable.AddressingStrategy.LINEAR);
 
             acquiredPages = new HashMap<>(16, 0.75f);
         }


[40/50] ignite git commit: ignite-db - free

Posted by sb...@apache.org.
ignite-db - free


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/956f083d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/956f083d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/956f083d

Branch: refs/heads/ignite-db-x-10884
Commit: 956f083d303d907c46320e43d8a96660bd6e352a
Parents: 8bb76a8
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 06:53:37 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 06:53:37 2016 +0300

----------------------------------------------------------------------
 .../processors/cache/database/MetadataStorage.java |  2 +-
 .../processors/cache/database/RowStore.java        |  1 +
 .../cache/database/freelist/FreeList.java          |  2 ++
 .../database/IgniteCacheH2DatabaseManager.java     | 17 +++++++++++------
 .../processors/query/h2/IgniteH2Indexing.java      | 10 ++++++++--
 5 files changed, 23 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/956f083d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
index bea27a4..169c97c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
@@ -160,7 +160,7 @@ public class MetadataStorage implements MetaStore {
             try {
                 long nextPageId = writeBuf.getLong();
 
-                assert nextPageId == 0;
+                assert nextPageId == 0: nextPageId;
 
                 // Position buffer to the last record.
                 writeBuf.position(state.position);

http://git-wip-us.apache.org/repos/asf/ignite/blob/956f083d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
index fdfadd6..a66e9dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
@@ -95,6 +95,7 @@ public class RowStore<T extends CacheDataRow> {
      */
     public RowStore(GridCacheContext<?,?> cctx, FreeList freeList) {
         assert cctx != null;
+        assert freeList != null;
 
         this.cctx = cctx;
         this.freeList = freeList;

http://git-wip-us.apache.org/repos/asf/ignite/blob/956f083d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
index 30ed519..721f0c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
@@ -61,6 +61,8 @@ public class FreeList {
 
             assert idx >= 0;
 
+            row.link(PageIdUtils.linkFromDwordOffset(page.id(), idx));
+
             int freeSpace = io.getFreeSpace(buf);
 
             // Put our free item.

http://git-wip-us.apache.org/repos/asf/ignite/blob/956f083d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheH2DatabaseManager.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheH2DatabaseManager.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheH2DatabaseManager.java
index aa94bfb..5debe92 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheH2DatabaseManager.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheH2DatabaseManager.java
@@ -76,11 +76,6 @@ public class IgniteCacheH2DatabaseManager extends GridCacheManagerAdapter implem
             if (primaryIdx != null)
                 throw new IgniteCheckedException("Primary index already exists for cache " +
                     "(make sure only one key-value type pair is stored in the cache): " + cctx.name());
-
-            int cpus = Runtime.getRuntime().availableProcessors();
-
-            reuseList = new ReuseList(cctx.cacheId(), dbMgr.pageMemory(), cpus * 2, dbMgr.meta());
-            freeList = new FreeList(cctx, reuseList);
         }
 
         Index idx = new H2TreeIndex(
@@ -106,7 +101,17 @@ public class IgniteCacheH2DatabaseManager extends GridCacheManagerAdapter implem
      * @param tbl Table.
      * @return New row store for the given table.
      */
-    public H2RowStore createRowStore(GridH2Table tbl) {
+    public H2RowStore createRowStore(GridH2Table tbl) throws IgniteCheckedException {
+        assert reuseList == null;
+        assert freeList == null;
+
+        int cpus = Runtime.getRuntime().availableProcessors();
+
+        IgniteCacheDatabaseSharedManager dbMgr = cctx.shared().database();
+
+        reuseList = new ReuseList(cctx.cacheId(), dbMgr.pageMemory(), cpus * 2, dbMgr.meta());
+        freeList = new FreeList(cctx, reuseList);
+
         return new H2RowStore(tbl.rowDescriptor(), cctx, freeList);
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/956f083d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 7d226ea..d48931d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1983,8 +1983,14 @@ public class IgniteH2Indexing implements GridQueryIndexing {
 
             IgniteCacheH2DatabaseManager dbMgr = databaseManager(cacheId);
 
-            if (dbMgr != null)
-                return dbMgr.createRowStore(tbl);
+            if (dbMgr != null) {
+                try {
+                    return dbMgr.createRowStore(tbl);
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException(e);
+                }
+            }
 
             return null;
         }


[34/50] ignite git commit: ignite-db - ok

Posted by sb...@apache.org.
ignite-db - ok


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3daefaea
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3daefaea
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3daefaea

Branch: refs/heads/ignite-db-x-10884
Commit: 3daefaea84a3db026e7cadc21b46b33f9e5f7bdb
Parents: 3166aeb
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 00:51:06 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 00:51:06 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          |  54 ++---
 .../processors/database/BPlusTreeSelfTest.java  | 214 +++++++++++++------
 2 files changed, 179 insertions(+), 89 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/3daefaea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index b90c5b2..06649cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -1308,7 +1308,7 @@ public abstract class BPlusTree<L, T extends L> {
 
                     case Put.NOT_FOUND: // Do insert.
                         assert lvl == p.btmLvl : "must insert at the bottom level";
-                        assert p.needReplaceInner == FALSE: p.needReplaceInner;
+                        assert p.needReplaceInner == FALSE: p.needReplaceInner + " " + lvl;
 
                         // Init args.
                         p.pageId = pageId;
@@ -1746,38 +1746,42 @@ public abstract class BPlusTree<L, T extends L> {
             assert !isFinished();
             assert tail.type == Tail.EXACT;
 
-            // Need to lock more to be able to merge.
-            if (needReplaceInner == TRUE ||
-                (needMergeEmptyBranch == TRUE && (tail.down == null || tail.getCount() == 0)))
-                return false;
+            boolean mergedBranch = false;
 
-            // Can merge only if tail is not a routing page.
-            if (tail.getCount() > 0) {
-                if (needMergeEmptyBranch == TRUE) {
-                    mergeEmptyBranch();
+            if (needMergeEmptyBranch == TRUE) {
+                // We can't merge empty branch if tail in routing page.
+                if (tail.down == null || tail.getCount() == 0)
+                    return false; // Lock the whole branch up to the first non-empty.
 
-                    needMergeEmptyBranch = DONE;
-                }
+                mergeEmptyBranch();
 
-                boolean mergeMore = mergeDown(tail);
+                mergedBranch = true;
+                needMergeEmptyBranch = DONE;
+            }
 
-                if (needReplaceInner == READY) {
-                    // If we've merged empty branch then the inner key was dropped.
-                    if (needMergeEmptyBranch != DONE)
-                        replaceInner(); // Need to replace inner key with new max key for the left subtree.
+            mergeDown(tail);
 
-                    needReplaceInner = DONE;
-                }
+            if (needReplaceInner == READY) {
+                // If we've merged empty branch right now, then the inner key was dropped.
+                if (!mergedBranch)
+                    replaceInner(); // Replace inner key with new max key for the left subtree.
 
-                if (tail.getCount() == 0 && tail.lvl != 0 && getRootLevel(meta) == tail.lvl)
-                    freePage(tail.page, tail.buf, tail.io, tail.lvl, false); // Free root.
-                else if (mergeMore) {
-                    doReleaseTail(tail.down);
+                needReplaceInner = DONE;
+            }
+            else if (needReplaceInner == TRUE)
+                return false; // Lock the whole branch up to the inner key page.
 
-                    tail.down = null;
+            if (tail.getCount() == 0 && tail.lvl != 0 && getRootLevel(meta) == tail.lvl) {
+                // Free root if it became empty after merge.
+                freePage(tail.page, tail.buf, tail.io, tail.lvl, false);
+            }
+            else if (tail.sibling != null && tail.getCount() + tail.sibling.getCount() < tail.io.getMaxCount(tail.buf)) {
+                // Release everything lower than tail, we've already merged this path.
+                doReleaseTail(tail.down);
 
-                    return false;
-                }
+                tail.down = null;
+
+                return false; // Lock and merge one level more.
             }
 
             releaseTail();

http://git-wip-us.apache.org/repos/asf/ignite/blob/3daefaea/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index c2b7a0b..eebd304 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -58,7 +58,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     private static final int CACHE_ID = 100500;
 
     /** */
-    private static int MAX_ITEMS_COUNT = 0;
+    private static int MAX_PER_PAGE = 0;
 
     /** */
     private static int CNT = 10;
@@ -84,7 +84,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     @Override protected void beforeTest() throws Exception {
         long seed = System.nanoTime();
 
-        X.println("Test seed: " + seed);
+        X.println("Test seed: " + seed + "L");
 
         TestTree.rnd = new Random(seed);
 
@@ -105,7 +105,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     @Override protected void afterTest() throws Exception {
         pageMem.stop();
 
-        MAX_ITEMS_COUNT = 0;
+        MAX_PER_PAGE = 0;
         PUT_INC = 1;
         RMV_INC = -1;
         CNT = 10;
@@ -115,7 +115,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_mm_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = -1;
         RMV_INC = -1;
@@ -127,7 +127,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_mm_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = -1;
         RMV_INC = -1;
@@ -139,7 +139,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_pm_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = 1;
         RMV_INC = -1;
@@ -151,7 +151,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_pm_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = 1;
         RMV_INC = -1;
@@ -163,7 +163,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_pp_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = 1;
         RMV_INC = 1;
@@ -175,7 +175,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_pp_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = 1;
         RMV_INC = 1;
@@ -187,7 +187,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_mp_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = -1;
         RMV_INC = 1;
@@ -199,7 +199,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_1_20_mp_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 1;
+        MAX_PER_PAGE = 1;
         CNT = 20;
         PUT_INC = -1;
         RMV_INC = 1;
@@ -212,7 +212,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_mm_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = -1;
         RMV_INC = -1;
@@ -224,7 +224,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_mm_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = -1;
         RMV_INC = -1;
@@ -236,7 +236,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_pm_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = 1;
         RMV_INC = -1;
@@ -248,7 +248,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_pm_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = 1;
         RMV_INC = -1;
@@ -260,7 +260,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_pp_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = 1;
         RMV_INC = 1;
@@ -272,7 +272,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_pp_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = 1;
         RMV_INC = 1;
@@ -284,7 +284,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_mp_1() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = -1;
         RMV_INC = 1;
@@ -296,7 +296,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testPutRemove_2_40_mp_0() throws IgniteCheckedException {
-        MAX_ITEMS_COUNT = 2;
+        MAX_PER_PAGE = 2;
         CNT = 40;
         PUT_INC = -1;
         RMV_INC = 1;
@@ -304,6 +304,103 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
         doTestPutRemove(false);
     }
 
+    // ------- 3 - 60
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_mm_1() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = -1;
+        RMV_INC = -1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_mm_0() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = -1;
+        RMV_INC = -1;
+
+        doTestPutRemove(false);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_pm_1() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = 1;
+        RMV_INC = -1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_pm_0() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = 1;
+        RMV_INC = -1;
+
+        doTestPutRemove(false);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_pp_1() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = 1;
+        RMV_INC = 1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_pp_0() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = 1;
+        RMV_INC = 1;
+
+        doTestPutRemove(false);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_mp_1() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = -1;
+        RMV_INC = 1;
+
+        doTestPutRemove(true);
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testPutRemove_3_60_mp_0() throws IgniteCheckedException {
+        MAX_PER_PAGE = 3;
+        CNT = 60;
+        PUT_INC = -1;
+        RMV_INC = 1;
+
+        doTestPutRemove(false);
+    }
+
     /**
      * @param canGetRow Can get row from inner page.
      * @throws IgniteCheckedException If failed.
@@ -346,16 +443,20 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void _testRandomRemove0() throws IgniteCheckedException {
-        // seed: 1461177795261173000, 1461187841179332000
+    public void testRandomRemove_1_30_0() throws IgniteCheckedException {
+        MAX_PER_PAGE = 1;
+        CNT = 30;
+
         doTestRandomRemove(false);
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void _testRandomRemove1() throws IgniteCheckedException {
-        // seed: 1461188744119034000 1461188844311788000 1461189099834526000
+    public void testRandomRemove_1_30_1() throws IgniteCheckedException {
+        MAX_PER_PAGE = 1;
+        CNT = 30;
+
         doTestRandomRemove(true);
     }
 
@@ -368,58 +469,43 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         Map<Long,Long> map = new HashMap<>();
 
-        int cnt = 100_000;
+        for (int i = 0 ; i < 100_000; i++) {
+            Long x = (long)tree.randomInt(CNT);
 
-        int rmv = 0;
+            if (i % 1000 == 0)
+                X.println(" --> " + i + "  " + x);
 
-        for (long x = 0; x < cnt; x++)
-            assertEquals(map.put(x,x), tree.put(x));
-
-        for (;;) {
-            for (int i = 0; i < 1000 && !map.isEmpty();) {
-                Long x = (long)tree.randomInt(cnt);
-
-                if (map.remove(x) != null) {
-//                    if (rmv > 93440) {
-//                        X.println("Rmv: " + rmv + " -> " + x);
-//
-//                        if (rmv == 93449)
-//                            X.println(tree.printTree());
-//                    }
+//            if (i >= 57)
+//                X.println(tree.printTree());
 
+            if (tree.randomInt(2) == 0)
+                assertEquals(map.put(x, x), tree.put(x));
+            else {
+                if (map.remove(x) != null)
                     assertEquals(x, tree.remove(x));
-                    assertNull(tree.remove(x));
-
-                    rmv++;
-                    i++;
-                }
+                assertNull(tree.remove(x));
             }
 
-            GridCursor<Long> cursor = tree.find(null, null);
-
-            int size = 0;
+            if (i % 100 == 0) {
+                GridCursor<Long> cursor = tree.find(null, null);
 
-            while (cursor.next()) {
-                size++;
+                int size = 0;
 
-                Long x = cursor.get();
+                while (cursor.next()) {
+                    size++;
 
-                assert x != null;
+                    x = cursor.get();
 
-                assertEquals(map.get(x), x);
-            }
+                    assert x != null;
 
-            assertEquals(map.size(), size);
+                    assertEquals(map.get(x), x);
+                }
 
-            if (size == 0)
-                break;
+                assertEquals(map.size(), size);
+            }
         }
     }
 
-    private void doTestStagedPutRemove(boolean canGetRow) {
-        // TODO
-    }
-
     /**
      * @param canGetRow Can get row from inner page.
      * @return Test tree instance.
@@ -540,8 +626,8 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         /** {@inheritDoc} */
         @Override public int getMaxCount(ByteBuffer buf) {
-            if (MAX_ITEMS_COUNT != 0)
-                return MAX_ITEMS_COUNT;
+            if (MAX_PER_PAGE != 0)
+                return MAX_PER_PAGE;
 
             return super.getMaxCount(buf);
         }
@@ -579,8 +665,8 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         /** {@inheritDoc} */
         @Override public int getMaxCount(ByteBuffer buf) {
-            if (MAX_ITEMS_COUNT != 0)
-                return MAX_ITEMS_COUNT;
+            if (MAX_PER_PAGE != 0)
+                return MAX_PER_PAGE;
 
             return super.getMaxCount(buf);
         }


[43/50] ignite git commit: Merge branch 'ignite-db-x' of https://github.com/gridgain/apache-ignite into ignite-db-2

Posted by sb...@apache.org.
Merge branch 'ignite-db-x' of https://github.com/gridgain/apache-ignite into ignite-db-2

# Conflicts:
#	modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5761012a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5761012a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5761012a

Branch: refs/heads/ignite-db-x-10884
Commit: 5761012af6ce4798c0fa94c86a4357a40cb02a6a
Parents: 50e452c 6d68db9
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 10:08:48 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 10:08:48 2016 +0300

----------------------------------------------------------------------
 .../internal/pagemem/DirectMemoryUtils.java     |  10 +
 .../ignite/internal/pagemem/FullPageId.java     |  44 +++-
 .../ignite/internal/pagemem/PageIdUtils.java    |  89 +++++---
 .../ignite/internal/pagemem/PageMemory.java     |   3 +-
 .../internal/pagemem/impl/FullPageIdTable.java  | 111 +++++++--
 .../ignite/internal/pagemem/impl/PageImpl.java  |   9 +
 .../internal/pagemem/impl/PageMemoryImpl.java   | 225 ++++++++++++++++++-
 .../pagemem/impl/PageIdUtilsSelfTest.java       |  39 ++++
 .../IgniteDbMultiNodePutGetSelfTest.java        |   5 +
 .../IgniteDbSingleNodePutGetSelfTest.java       | 120 ++++++++++
 10 files changed, 578 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/5761012a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
index 1e1fb78,28d4a19..91e7798
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
@@@ -161,12 -165,16 +165,14 @@@ class PageImpl extends AbstractQueuedSy
  
      /** {@inheritDoc} */
      @Override public ByteBuffer getForWrite() {
 -        Thread th = Thread.currentThread();
 +        acquire(1); // This call is not reentrant.
  
 -        if (getExclusiveOwnerThread() != th) {
 -            acquire(1);
 +        assert getExclusiveOwnerThread() == null: fullId();
  
 -            setExclusiveOwnerThread(th);
 +        setExclusiveOwnerThread(Thread.currentThread());
  
 -            pageMem.writeCurrentTimestamp(ptr);
 -        }
++        pageMem.writeCurrentTimestamp(ptr);
+ 
          return reset(buf);
      }
  

http://git-wip-us.apache.org/repos/asf/ignite/blob/5761012a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
----------------------------------------------------------------------


[19/50] ignite git commit: GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.

Posted by sb...@apache.org.
GG-11044 - Implement page evictions from PageMemory for the case when data size is bigger than allocated memory.

Review fixes:
- Replace BitSet with HashSet.
- Remove array allocation.
- Replace atomic write to volatile write for timestamp.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d7dfb91b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d7dfb91b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d7dfb91b

Branch: refs/heads/ignite-db-x-10884
Commit: d7dfb91ba6ec8c494f3776b8892b9923c8e96ed5
Parents: 36b0a20
Author: dkarachentsev <dk...@gridgain.com>
Authored: Mon Apr 25 10:36:13 2016 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Mon Apr 25 10:36:13 2016 +0300

----------------------------------------------------------------------
 .../internal/pagemem/impl/PageMemoryImpl.java     | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/d7dfb91b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
index 5075f6a..7863fbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
@@ -956,6 +956,8 @@ public class  PageMemoryImpl implements PageMemory {
 
         long relEvictAddr = INVALID_REL_PTR;
 
+        int iterations = 0;
+
         while (true) {
             long cleanAddr = INVALID_REL_PTR;
             long cleanTs = Long.MAX_VALUE;
@@ -1004,10 +1006,12 @@ public class  PageMemoryImpl implements PageMemory {
             final long metaPageId = mem.readLong(dbMetaPageIdPtr);
 
             if (fullPageId.pageId() == metaPageId && fullPageId.cacheId() == 0) {
-                if (ignored == null)
-                    ignored = new HashSet<>();
+                if (++iterations > 2) {
+                    if (ignored == null)
+                        ignored = new HashSet<>();
 
-                ignored.add(relEvictAddr);
+                    ignored.add(relEvictAddr);
+                }
 
                 continue;
             }
@@ -1017,10 +1021,12 @@ public class  PageMemoryImpl implements PageMemory {
             if (!seg.acquiredPages.containsKey(fullPageId))
                 seg.loadedPages.remove(fullPageId);
             else {
-                if (ignored == null)
-                    ignored = new HashSet<>();
+                if (++iterations > 2) {
+                    if (ignored == null)
+                        ignored = new HashSet<>();
 
-                ignored.add(relEvictAddr);
+                    ignored.add(relEvictAddr);
+                }
 
                 continue;
             }


[33/50] ignite git commit: Merge branch ignite-gg-11044 into ignite-db-x

Posted by sb...@apache.org.
Merge branch ignite-gg-11044 into ignite-db-x


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6d68db94
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6d68db94
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6d68db94

Branch: refs/heads/ignite-db-x-10884
Commit: 6d68db94fe9b6a36a082c7f350adf2e2da2732c9
Parents: a358116 2da9ee3
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 26 11:10:11 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 26 11:10:11 2016 -0700

----------------------------------------------------------------------
 .../internal/pagemem/DirectMemoryUtils.java     |  10 +
 .../ignite/internal/pagemem/PageMemory.java     |   3 +-
 .../internal/pagemem/impl/FullPageIdTable.java  |  90 +++++++-
 .../ignite/internal/pagemem/impl/PageImpl.java  |   9 +
 .../internal/pagemem/impl/PageMemoryImpl.java   | 225 ++++++++++++++++++-
 5 files changed, 316 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/6d68db94/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
index 9c4ce66,c6529f4..2aa53c6
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
@@@ -175,8 -223,8 +220,8 @@@ public class FullPageIdTable 
          do {
              int res = testKeyAt(index, key);
  
-             if (res == EMPTY) {
+             if (res == EMPTY || res == REMOVED) {
 -                setKeyAt(index, key);
 +                setKeyAt(index, key.pageId(), key.cacheId());
  
                  incrementSize();
  


[46/50] ignite git commit: Merge branch ignite-db-x into ignite-db-2

Posted by sb...@apache.org.
Merge branch ignite-db-x into ignite-db-2


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/cf5ad2bc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/cf5ad2bc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/cf5ad2bc

Branch: refs/heads/ignite-db-x-10884
Commit: cf5ad2bc7139d58364eccfe302d07b8943e28fbd
Parents: d67fa20 6d68db9
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Apr 27 00:49:06 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 27 00:49:06 2016 -0700

----------------------------------------------------------------------
 .../internal/pagemem/DirectMemoryUtils.java     |  10 +
 .../ignite/internal/pagemem/FullPageId.java     |  44 +++-
 .../ignite/internal/pagemem/PageIdUtils.java    |  89 +++++---
 .../ignite/internal/pagemem/PageMemory.java     |   3 +-
 .../internal/pagemem/impl/FullPageIdTable.java  | 111 +++++++--
 .../ignite/internal/pagemem/impl/PageImpl.java  |   9 +
 .../internal/pagemem/impl/PageMemoryImpl.java   | 225 ++++++++++++++++++-
 .../pagemem/impl/PageIdUtilsSelfTest.java       |  39 ++++
 .../IgniteDbMultiNodePutGetSelfTest.java        |   5 +
 .../IgniteDbSingleNodePutGetSelfTest.java       | 120 ++++++++++
 10 files changed, 578 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/cf5ad2bc/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
index 1e1fb78,28d4a19..91e7798
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
@@@ -161,12 -165,16 +165,14 @@@ class PageImpl extends AbstractQueuedSy
  
      /** {@inheritDoc} */
      @Override public ByteBuffer getForWrite() {
 -        Thread th = Thread.currentThread();
 +        acquire(1); // This call is not reentrant.
  
 -        if (getExclusiveOwnerThread() != th) {
 -            acquire(1);
 +        assert getExclusiveOwnerThread() == null: fullId();
  
 -            setExclusiveOwnerThread(th);
 +        setExclusiveOwnerThread(Thread.currentThread());
  
 -            pageMem.writeCurrentTimestamp(ptr);
 -        }
++        pageMem.writeCurrentTimestamp(ptr);
+ 
          return reset(buf);
      }
  

http://git-wip-us.apache.org/repos/asf/ignite/blob/cf5ad2bc/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
index 09e0df8,4422962..7611675
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
@@@ -214,16 -223,18 +229,21 @@@ public class  PageMemoryImpl implement
                      break;
              }
  
 +            // Assign page ID according to flags and partition ID.
 +            pageId = PageIdUtils.pageId(partId, flags, pageId);
 +
              relPtr = allocateFreePage();
  
+             if (relPtr == INVALID_REL_PTR)
+                 throw new OutOfMemoryException();
+ 
              absPtr = absolute(relPtr);
 -
 -            writePageId(absPtr, pageId);
 -            writePageCacheId(absPtr, cacheId);
 -            writeCurrentTimestamp(absPtr);
          }
  
 +        writePageId(absPtr, pageId);
++        writePageCacheId(absPtr, cacheId);
++        writeCurrentTimestamp(absPtr);
 +
          // TODO pass an argument to decide whether the page should be cleaned.
          mem.setMemory(absPtr + PAGE_OVERHEAD, sysPageSize - PAGE_OVERHEAD, (byte)0);
  

http://git-wip-us.apache.org/repos/asf/ignite/blob/cf5ad2bc/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
----------------------------------------------------------------------


[23/50] ignite git commit: Added test.

Posted by sb...@apache.org.
Added test.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/823fabd9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/823fabd9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/823fabd9

Branch: refs/heads/ignite-db-x-10884
Commit: 823fabd94052fd04e748faffa42e7417c3024663
Parents: bb79688
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Apr 25 16:37:47 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Apr 25 16:37:47 2016 -0700

----------------------------------------------------------------------
 .../IgniteDbSingleNodePutGetSelfTest.java       | 58 +++++++++++++++++++-
 1 file changed, 56 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/823fabd9/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
index 7b7d791..1bb95fa 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
@@ -274,6 +274,62 @@ public class IgniteDbSingleNodePutGetSelfTest extends GridCommonAbstractTest {
     /**
      * @throws Exception if failed.
      */
+    public void testBounds() throws Exception {
+        IgniteEx ig = grid(0);
+
+        final IgniteCache<Integer, DbValue> cache = ig.cache(null);
+
+        X.println("Put start");
+
+        int cnt = 1000;
+
+        try (IgniteDataStreamer<Integer, DbValue> st = ig.dataStreamer(null)) {
+            st.allowOverwrite(true);
+
+            for (int i = 0; i < cnt; i++) {
+                int k = 2 * i;
+
+                DbValue v0 = new DbValue(k, "test-value", k);
+
+                st.addData(k, v0);
+            }
+        }
+
+        X.println("Get start");
+
+        for (int i = 0; i < cnt; i++) {
+            int k = 2 * i;
+
+            DbValue v0 = new DbValue(k, "test-value", k);
+
+            assertEquals(v0, cache.get(k));
+        }
+
+        awaitPartitionMapExchange();
+
+        X.println("Query start");
+
+        // Make sure to cover multiple pages.
+        int limit = 500;
+
+        for (int i = 0; i < limit; i++) {
+            List<List<?>> res = cache.query(new SqlFieldsQuery("select ival, _val from dbvalue where ival < ? order by ival")
+                .setArgs(i)).getAll();
+
+            // 0 => 0, 1 => 1, 2=>1,...
+            assertEquals((i + 1) / 2, res.size());
+
+            res = cache.query(new SqlFieldsQuery("select ival, _val from dbvalue where ival <= ? order by ival")
+                .setArgs(i)).getAll();
+
+            // 0 => 1, 1 => 1, 2=>2,...
+            assertEquals(i / 2 + 1, res.size());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
     public void testMultithreadedPut() throws Exception {
         IgniteEx ig = grid(0);
 
@@ -331,8 +387,6 @@ public class IgniteDbSingleNodePutGetSelfTest extends GridCommonAbstractTest {
             "explain select lval from dbvalue where ival >= 5000 and ival < 7000")).getAll().get(0).get(0).toString();
 
         assertTrue(plan, plan.contains("iVal_idx"));
-
-        cache.clear();
     }
 
     /**


[15/50] ignite git commit: ignite-db - split fix

Posted by sb...@apache.org.
ignite-db - split fix


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d9a3a82f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d9a3a82f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d9a3a82f

Branch: refs/heads/ignite-db-x-10884
Commit: d9a3a82fbe6aaba78fa2adda476070ba8031c224
Parents: 29179ce
Author: S.Vladykin <sv...@gridgain.com>
Authored: Sun Apr 24 21:13:16 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Sun Apr 24 21:13:16 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/d9a3a82f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 05fa56b..aa54554 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -1043,12 +1043,22 @@ public abstract class BPlusTree<L, T extends L> {
      * @param io IO.
      * @param buf Splitting buffer.
      * @param fwdBuf Forward buffer.
+     * @param idx Insertion index.
+     * @return {@code true} The middle index was shifted to the right.
      * @throws IgniteCheckedException If failed.
      */
-    private void splitPage(BPlusIO io, ByteBuffer buf, ByteBuffer fwdBuf)
+    private boolean splitPage(BPlusIO io, ByteBuffer buf, ByteBuffer fwdBuf, int idx)
         throws IgniteCheckedException {
         int cnt = io.getCount(buf);
-        int mid = 1 + (cnt >>> 1);
+        int mid = cnt >>> 1;
+
+        boolean res = false;
+
+        if (idx > mid) { // If insertion is going to be to the forward page, keep more in the back page.
+            mid++;
+
+            res = true;
+        }
 
         cnt -= mid;
 
@@ -1061,6 +1071,8 @@ public abstract class BPlusTree<L, T extends L> {
         // Setup forward-backward refs.
         io.setForward(fwdBuf, io.getForward(buf));
         io.setForward(buf, PageIO.getPageId(fwdBuf));
+
+        return res;
     }
 
     /**
@@ -1105,12 +1117,12 @@ public abstract class BPlusTree<L, T extends L> {
             ByteBuffer fwdBuf = fwd.getForInitialWrite();
             io.initNewPage(fwdBuf, fwd.id());
 
-            splitPage(io, buf, fwdBuf);
+            boolean midShift = splitPage(io, buf, fwdBuf, idx);
 
             // Do insert.
             int cnt = io.getCount(buf);
 
-            if (idx <= cnt) {
+            if (idx < cnt || (idx == cnt && !midShift)) {
                 insertSimple(io, buf, row, idx, rightId);
 
                 // Fix leftmost child of forward page, because newly inserted row will go up.


[44/50] ignite git commit: Fixed page ID allocation in case when no store is present.

Posted by sb...@apache.org.
Fixed page ID allocation in case when no store is present.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/66fb8ef0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/66fb8ef0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/66fb8ef0

Branch: refs/heads/ignite-db-x-10884
Commit: 66fb8ef05ffacedacf01e5d800f54224783b8454
Parents: ca839c8
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Apr 27 00:32:54 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 27 00:32:54 2016 -0700

----------------------------------------------------------------------
 .../ignite/internal/pagemem/impl/PageMemoryImpl.java  | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/66fb8ef0/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
index b8d58f6..09e0df8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
@@ -38,6 +38,7 @@ import org.apache.ignite.internal.mem.OutOfMemoryException;
 import org.apache.ignite.internal.pagemem.DirectMemoryUtils;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.Page;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
@@ -199,6 +200,11 @@ public class  PageMemoryImpl implements PageMemory {
             absPtr = absolute(relPtr);
 
             pageId = readPageId(absPtr);
+
+            long idx = PageIdUtils.pageIdx(pageId);
+
+            // Reassign page ID according to flags and partition ID.
+            pageId = PageIdUtils.pageId(partId, flags, idx);
         }
         else {
             while (true) {
@@ -208,13 +214,16 @@ public class  PageMemoryImpl implements PageMemory {
                     break;
             }
 
+            // Assign page ID according to flags and partition ID.
+            pageId = PageIdUtils.pageId(partId, flags, pageId);
+
             relPtr = allocateFreePage();
 
             absPtr = absolute(relPtr);
-
-            writePageId(absPtr, pageId);
         }
 
+        writePageId(absPtr, pageId);
+
         // TODO pass an argument to decide whether the page should be cleaned.
         mem.setMemory(absPtr + PAGE_OVERHEAD, sysPageSize - PAGE_OVERHEAD, (byte)0);
 
@@ -234,7 +243,6 @@ public class  PageMemoryImpl implements PageMemory {
         return fullId;
     }
 
-
     /** {@inheritDoc} */
     @Override public boolean freePage(FullPageId fullId) throws IgniteCheckedException {
         Segment seg = segment(fullId);


[21/50] ignite git commit: ignite-db - wip

Posted by sb...@apache.org.
ignite-db - wip


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/74dfb8f9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/74dfb8f9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/74dfb8f9

Branch: refs/heads/ignite-db-x-10884
Commit: 74dfb8f971cd44a1b10d8fdab6a8bb19c98dd7a7
Parents: 3df530a
Author: S.Vladykin <sv...@gridgain.com>
Authored: Mon Apr 25 19:58:51 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Mon Apr 25 19:58:51 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 325 +++++++------------
 .../processors/database/BPlusTreeSelfTest.java  |  89 ++++-
 2 files changed, 202 insertions(+), 212 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/74dfb8f9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 61e612a..a24c38a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -210,8 +210,10 @@ public abstract class BPlusTree<L, T extends L> {
 
                 if (cnt != 0) // It is not a routing page and we are going to the right, can get backId here.
                     g.backId = inner(io).getLeft(buf, cnt - 1);
-                else if (needBackIfRouting && g.getClass() == Remove.class)
-                    return Remove.GO_DOWN_X; // Can't get backId here because of possible deadlock.
+                else if (needBackIfRouting) {
+                    // Can't get backId here because of possible deadlock and it is only needed for remove operation.
+                    return Remove.GO_DOWN_X;
+                }
             }
 
             return Get.GO_DOWN;
@@ -322,7 +324,16 @@ public abstract class BPlusTree<L, T extends L> {
 
             r.removed = getRow(io, buf, idx);
 
-            r.doRemove(io, leaf, buf, cnt, idx, 0, false);
+            r.doRemove(io, buf, cnt, idx);
+
+            if (r.needReplaceInner == TRUE) {
+                // We increment remove ID in write lock on leaf page, thus it is guaranteed that
+                // any successor will get greater value than he had read at the beginning of the operation.
+                // Thus he is guaranteed to do a retry from root. Since inner replace takes locks on the whole branch
+                // and releases the locks only when the inner key is updated and the successor saw the updated removeId,
+                // then after retry from root, he will see updated inner key.
+                io.setRemoveId(buf, globalRmvId.incrementAndGet());
+            }
 
             // We may need to replace inner key or want to merge this leaf with sibling after the remove -> keep lock.
             if (r.needReplaceInner == TRUE ||
@@ -332,10 +343,7 @@ public abstract class BPlusTree<L, T extends L> {
                 if (r.fwdId != 0 && r.backId == 0)
                     r.lockForward(0);
 
-                r.addTail(leaf, buf, io, 0, Tail.EXACT, Integer.MIN_VALUE);
-
-                if (r.needReplaceInner == FALSE)
-                    r.needMerge = TRUE;
+                r.addTail(leaf, buf, io, 0, Tail.EXACT, -1);
             }
 
             return Remove.FOUND;
@@ -353,12 +361,9 @@ public abstract class BPlusTree<L, T extends L> {
             // Correct locking order: from back to forward.
             int res = r.doRemoveFromLeaf();
 
-            // If we need to do more tricks, then we have to keep locks on back and leaf pages.
-            if (res == Remove.FOUND && (r.needMerge == TRUE || r.needReplaceInner == TRUE)) {
-                assert r.needMerge == TRUE ^ r.needReplaceInner == TRUE: "we can do only one thing at once";
-
-                r.addTail(back, buf, io, lvl, Tail.BACK, Integer.MIN_VALUE);
-            }
+            // Keep locks on back and leaf pages for subsequent merges.
+            if (res == Remove.FOUND && r.tail != null)
+                r.addTail(back, buf, io, lvl, Tail.BACK, -1);
 
             return res;
         }
@@ -376,7 +381,7 @@ public abstract class BPlusTree<L, T extends L> {
             int res = r.doLockTail(lvl);
 
             if (res == Remove.FOUND)
-                r.addTail(back, buf, io, lvl, Tail.BACK, Integer.MIN_VALUE);
+                r.addTail(back, buf, io, lvl, Tail.BACK, -1);
 
             return res;
         }
@@ -387,7 +392,7 @@ public abstract class BPlusTree<L, T extends L> {
         @Override protected int run0(Page page, ByteBuffer buf, BPlusIO<L> io, Remove r, int lvl)
             throws IgniteCheckedException {
 
-            r.addTail(page, buf, io, lvl, Tail.FORWARD, Integer.MIN_VALUE);
+            r.addTail(page, buf, io, lvl, Tail.FORWARD, -1);
 
             return Remove.FOUND;
         }
@@ -435,13 +440,6 @@ public abstract class BPlusTree<L, T extends L> {
 
             r.addTail(page, buf, io, lvl, Tail.EXACT, idx);
 
-            if (r.needMerge == TRUE) {
-                assert r.needReplaceInner == FALSE;
-                assert r.tail.down.type == Tail.EXACT;
-
-                r.needMerge = READY;
-            }
-
             return Remove.FOUND;
         }
     };
@@ -547,7 +545,7 @@ public abstract class BPlusTree<L, T extends L> {
 
     /**
      * @param meta Meta page.
-     * @param lvl Level, if {@code 0} then it is a bottom level, if {@link Integer#MIN_VALUE}, then root.
+     * @param lvl Level, if {@code 0} then it is a bottom level, if negative then root.
      * @return Page ID.
      */
     private long getFirstPageId(Page meta, int lvl) {
@@ -556,7 +554,7 @@ public abstract class BPlusTree<L, T extends L> {
         try {
             BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(buf);
 
-            if (lvl == Integer.MIN_VALUE)
+            if (lvl < 0)
                 lvl = io.getRootLevel(buf);
 
             if (lvl >= io.getLevelsCount(buf))
@@ -673,6 +671,7 @@ public abstract class BPlusTree<L, T extends L> {
 
                 switch (res) {
                     case Get.GO_DOWN:
+                    case Get.GO_DOWN_X:
                         assert g.pageId != pageId;
                         assert g.fwdId != fwdId || fwdId == 0;
 
@@ -717,7 +716,7 @@ public abstract class BPlusTree<L, T extends L> {
         long rootPageId;
 
         try (Page meta = page(metaPageId)) {
-            rootPageId = getFirstPageId(meta, Integer.MIN_VALUE);
+            rootPageId = getFirstPageId(meta, -1);
         }
         catch (IgniteCheckedException e) {
             throw new IllegalStateException(e);
@@ -848,7 +847,7 @@ public abstract class BPlusTree<L, T extends L> {
 
                     default:
                         if (!r.isFinished())
-                            r.finishTail(true);
+                            r.finishTail();
 
                         assert r.isFinished();
 
@@ -910,7 +909,7 @@ public abstract class BPlusTree<L, T extends L> {
                                 return res;
                         }
 
-                        if (!r.isFinished() && !r.finishTail(false))
+                        if (!r.isFinished() && !r.finishTail())
                             return r.lockTail(page, backId, fwdId, lvl);
 
                         return res;
@@ -939,7 +938,7 @@ public abstract class BPlusTree<L, T extends L> {
 
                             r.finish();
                         }
-                        else if (res == Remove.FOUND && r.needReplaceInner == FALSE && r.needMerge == FALSE) {
+                        else if (res == Remove.FOUND && r.tail == null) {
                             // Finish if we don't need to do any merges.
                             r.finish();
                         }
@@ -996,25 +995,6 @@ public abstract class BPlusTree<L, T extends L> {
      }
 
     /**
-     * @param cur Current tail element.
-     * @param fwdCnt Count in forward page.
-     * @return Count after merge or {@code -1} if merge is impossible.
-     */
-    private int countAfterMerge(Tail cur, int fwdCnt) {
-        int cnt = cur.io.getCount(cur.buf);
-
-        int newCnt = cnt + fwdCnt;
-
-        if (cur.lvl != 0)
-            newCnt++; // We have to move down split key in inner pages.
-
-        if (newCnt <= cur.io.getMaxCount(cur.buf))
-            return newCnt;
-
-        return -1;
-    }
-
-    /**
      * @param row Row.
      * @return Old row.
      * @throws IgniteCheckedException If failed.
@@ -1269,19 +1249,21 @@ public abstract class BPlusTree<L, T extends L> {
 
                 switch (res) {
                     case Put.GO_DOWN:
+                    case Put.GO_DOWN_X:
                         assert lvl > 0 : lvl;
                         assert p.pageId != pageId;
                         assert p.fwdId != fwdId || fwdId == 0;
 
-                        if (p.foundInner == TRUE) { // Need to replace ref in inner page.
-                            p.foundInner = FALSE; // Protect from retries.
+                        // TODO race on go down?
+                        if (p.needReplaceInner == TRUE) { // Need to replace ref in inner page.
+                            p.needReplaceInner = FALSE; // Protect from retries.
 
                             res = writePage(page, replace, p, lvl, Put.RETRY);
 
                             if (res != Put.FOUND)
                                 return res; // Need to retry.
 
-                            p.foundInner = DONE; // We can have only single matching inner key.
+                            p.needReplaceInner = DONE; // We can have only single matching inner key.
                         }
 
                         // Go down recursively.
@@ -1306,7 +1288,7 @@ public abstract class BPlusTree<L, T extends L> {
 
                     case Put.NOT_FOUND: // Do insert.
                         assert lvl == p.btmLvl : "must insert at the bottom level";
-                        assert p.foundInner == FALSE: p.foundInner;
+                        assert p.needReplaceInner == FALSE: p.needReplaceInner;
 
                         // Init args.
                         p.pageId = pageId;
@@ -1554,7 +1536,7 @@ public abstract class BPlusTree<L, T extends L> {
         short btmLvl;
 
         /** */
-        byte foundInner = FALSE;
+        byte needReplaceInner = FALSE;
 
         /**
          * @param row Row.
@@ -1569,8 +1551,8 @@ public abstract class BPlusTree<L, T extends L> {
                 return true;
 
             // If we can get full row from the inner page, we must do inner replace to update full row info here.
-            if (io.canGetRow() && foundInner == FALSE)
-                foundInner = TRUE;
+            if (io.canGetRow() && needReplaceInner == FALSE)
+                needReplaceInner = TRUE;
 
             return false;
         }
@@ -1631,9 +1613,6 @@ public abstract class BPlusTree<L, T extends L> {
         /** */
         byte needReplaceInner = FALSE;
 
-        /** */
-        byte needMerge = FALSE;
-
         /** Removed row. */
         T removed;
 
@@ -1688,49 +1667,33 @@ public abstract class BPlusTree<L, T extends L> {
         /**
          * Process tail and finish.
          *
-         * @param ignoreMergeMore Ignore the attempt to merge more pages up.
          * @return {@code false} If failed to finish and we need to lock more pages up.
          * @throws IgniteCheckedException If failed.
          */
-        private boolean finishTail(boolean ignoreMergeMore) throws IgniteCheckedException {
+        private boolean finishTail() throws IgniteCheckedException {
             assert !isFinished();
-            assert needMerge != FALSE || needReplaceInner != FALSE;
-            assert tail != null;
+            assert tail.type == Tail.EXACT;
 
-            if (needReplaceInner == TRUE)
-                return false; // Need to find inner page.
+            // Need to lock more.
+            if (tail.down == null || needReplaceInner == TRUE || tail.io.getCount(tail.buf) == 0)
+                return false;
 
-            if (needReplaceInner == READY) {
-                assert needMerge == FALSE: needMerge;
-
-                // We increment remove ID in write lock on leaf page, thus it is guaranteed that
-                // any successor will get greater value than he had read at the beginning of the operation.
-                // Thus he is guaranteed to do a retry from root. Since inner replace takes locks on the whole branch
-                // and releases the locks only when the inner key is updated and the successor saw the updated removeId,
-                // then after retry from root, he will see updated inner key.
-                globalRmvId.incrementAndGet();
+            // Merge from the top to the bottom.
+            for (Tail<L> t = tail; t.down != null; t = t.down)
+                merge(t);
 
+            if (needReplaceInner == READY) {
                 // Need to replace inner key with new max key for the left subtree.
-                doReplaceInner();
+                replaceInner();
 
                 needReplaceInner = DONE;
             }
-            else if (needMerge == READY) {
-                assert needReplaceInner == FALSE: needReplaceInner;
-                assert tail.down != null;
-
-                boolean needMergeMore = merge(tail.lvl - 1, true);
 
-                if (needMergeMore && !ignoreMergeMore) {
-                    needMerge = TRUE;
+//            if (cnt == 0 && lvl != 0 && getRootLevel(meta) == lvl) TODO
+//                freePage(page, buf, io, lvl); // Free root.
 
-                    return false;
-                }
-
-                needMerge = DONE;
-            }
-            else
-                return false;
+//            if (needMergeMore)
+//                return false;
 
             releaseTail();
             finish();
@@ -1798,7 +1761,7 @@ public abstract class BPlusTree<L, T extends L> {
          * @throws IgniteCheckedException If failed.
          */
         private int lockTail(Page page, long backId, long fwdId, int lvl) throws IgniteCheckedException {
-            assert needMerge == TRUE ^ needReplaceInner == TRUE: "we can do only one thing at once";
+            assert tail != null;
 
             // Init parameters for the handlers.
             this.pageId = page.id();
@@ -1839,53 +1802,49 @@ public abstract class BPlusTree<L, T extends L> {
          * @param buf Buffer.
          * @param cnt Count.
          * @param idx Index to remove.
-         * @param lvl Level.
-         * @param kickLeftChild If we are dropping left child instead of the right one.
          * @throws IgniteCheckedException If failed.
          */
-        private void doRemove(BPlusIO io, Page page, ByteBuffer buf, int cnt, int idx, int lvl, boolean kickLeftChild)
+        private void doRemove(BPlusIO io, ByteBuffer buf, int cnt, int idx)
             throws IgniteCheckedException {
-            assert cnt > 0;
-            assert idx >= 0;
-            assert idx < cnt; // TODO check why is the code below exists
-
-//            if (idx == cnt) {
-//                idx--; // This may happen in case of right turn, we need to remove the rightmost ref and link.
-//
-//                assert !kickLeftChild: "right child must be dropped here";
-//            }
+            assert cnt > 0: cnt;
+            assert idx >= 0 && idx < cnt: idx + " " + cnt;
 
             cnt--;
 
-            io.copyItems(buf, buf, idx + 1, idx, cnt - idx, kickLeftChild);
+            io.copyItems(buf, buf, idx + 1, idx, cnt - idx, false);
             io.setCount(buf, cnt);
-
-            if (cnt == 0 && lvl != 0 && getRootLevel(meta) == lvl)
-                freePage(page, buf, io, lvl); // Free root.
         }
 
         /**
          * @param prnt Parent tail.
-         * @param cur Current tail.
-         * @param fwd Forward tail.
+         * @param left Left child tail.
+         * @param right Right child tail.
+         * @return {@code true} If merged successfully.
          * @throws IgniteCheckedException If failed.
          */
-        private boolean mergePages(Tail<L> prnt, Tail<L> cur, Tail<L> fwd) throws IgniteCheckedException {
-            assert fwd.io == cur.io; // Otherwise incompatible.
+        private boolean mergePages(Tail<L> prnt, Tail<L> left, Tail<L> right) throws IgniteCheckedException {
+            assert right.io == left.io; // Otherwise incompatible.
+            assert left.io.getForward(left.buf) == right.page.id();
 
-            int cnt = cur.io.getCount(cur.buf);
-            int fwdCnt = fwd.io.getCount(fwd.buf);
-            int newCnt = countAfterMerge(cur, fwdCnt);
+            int prntCnt = prnt.io.getCount(prnt.buf);
+            int backCnt = left.io.getCount(left.buf);
+            int curCnt = right.io.getCount(right.buf);
 
-            if (newCnt == -1) // Not enough space.
-                return false;
+            assert prntCnt > 0: prntCnt;
 
-            cur.io.setCount(cur.buf, newCnt);
+            int newCnt = backCnt + curCnt;
 
-            int prntCnt = prnt.io.getCount(prnt.buf);
+            // Need to move down split key in inner pages, but for the last key it will be a special case merge.
+            if (left.lvl != 0 && false) // TODO
+                newCnt++;
+
+            if (newCnt > left.io.getMaxCount(left.buf))
+                return false;
+
+            left.io.setCount(left.buf, newCnt);
 
             // Move down split key in inner pages.
-            if (cur.lvl != 0) {
+            if (left.lvl != 0) {
                 int prntIdx = prnt.idx;
 
                 if (prntIdx == prntCnt) // It was a right turn.
@@ -1893,21 +1852,22 @@ public abstract class BPlusTree<L, T extends L> {
 
                 // We can be sure that we have enough free space to store split key here,
                 // because we've done remove already and did not release child locks.
-                inner(cur.io).store(cur.buf, cnt, prnt.io, prnt.buf, prntIdx);
+                left.io.store(left.buf, backCnt, prnt.io, prnt.buf, prntIdx);
 
-                cnt++;
+                backCnt++;
             }
 
-            cur.io.copyItems(fwd.buf, cur.buf, 0, cnt, fwdCnt, true);
-            cur.io.setForward(cur.buf, fwd.io.getForward(fwd.buf));
+            left.io.copyItems(right.buf, left.buf, 0, backCnt, curCnt, true);
+            left.io.setForward(left.buf, right.io.getForward(right.buf));
 
-            assert prntCnt > 0: prntCnt;
+            // Fix index for the right move: remove last item.
+            int prntIdx = prnt.idx == prntCnt ? prntCnt - 1 : prnt.idx;
 
-            // Remove split key from parent. If parent is root and becomes empty, it will be freed by doRemove.
-            doRemove(prnt.io, prnt.page, prnt.buf, prntCnt, prnt.idx, prnt.lvl, false);
+            // Remove split key from parent.
+            doRemove(prnt.io, prnt.buf, prntCnt, prntIdx);
 
             // Forward page is now empty and has no links.
-            freePage(fwd.page, fwd.buf, fwd.io, fwd.lvl);
+            freePage(right.page, right.buf, right.io, right.lvl);
 
             return true;
         }
@@ -1939,6 +1899,8 @@ public abstract class BPlusTree<L, T extends L> {
                 emptyPages = new ArrayList<>(4);
 
             emptyPages.add(page.fullId());
+
+            writeUnlockAndClose(page);
         }
 
         /**
@@ -1952,38 +1914,9 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
-         * @param inner Inner replace page.
          * @throws IgniteCheckedException If failed.
          */
-        private void dropEmptyBranch(Tail inner) throws IgniteCheckedException {
-            int cnt = inner.io.getCount(inner.buf);
-
-            assert cnt > 0: cnt;
-
-            // We need to check if the branch we are going to drop goes to the left or to the right.
-            boolean kickLeft = inner.down.page.id() == inner(inner.io).getLeft(inner.buf, inner.idx);
-
-            assert kickLeft || inner.down.page.id() == inner(inner.io).getRight(inner.buf, inner.idx);
-
-            // Remove found inner key from inner page.
-            doRemove(inner.io, inner.page, inner.buf, cnt, inner.idx, inner.lvl, kickLeft);
-
-            // If inner page was root and became empty, it was handled in doRemove.
-            // Otherwise we can be sure that inner page was not freed, at lead it must become
-            // an empty routing page. Thus always starting from inner.down here.
-            for (Tail t = inner.down; t != null; t = t.down) {
-                // TODO merge branch but not just free it
-
-                assert t.io.getCount(t.buf) == 0: row;
-
-                freePage(t.page, t.buf, t.io, t.lvl);
-            }
-        }
-
-        /**
-         * @throws IgniteCheckedException If failed.
-         */
-        private void doReplaceInner() throws IgniteCheckedException {
+        private void replaceInner() throws IgniteCheckedException {
             assert needReplaceInner == READY: needReplaceInner;
             assert tail.lvl > 0;
             assert innerIdx >= 0;
@@ -1996,89 +1929,59 @@ public abstract class BPlusTree<L, T extends L> {
 
             int cnt = leaf.io.getCount(leaf.buf);
 
-            if (cnt == 0) { // Merge empty leaf page.
-                if (!merge(0, false)) {
-                    // For leaf pages this can happen only when parent is empty -> drop the whole branch.
-                    dropEmptyBranch(inner);
-
-                    return;
-                }
-
-                // Need to handle possible tail restructuring after merge.
-                leaf = getTail(0);
-
-                cnt = leaf.io.getCount(leaf.buf);
+            assert cnt > 0: cnt; // Leaf must be merged at this point already if it was empty.
 
-                // If any leaf becomes empty we have to either merge it or drop the whole empty branch.
-                assert cnt > 0: "leaf can't be empty after successful merge";
-            }
-
-            // If after leaf merge parent have lost inner key, we don't need to update it anymore.
             if (innerIdx < inner.io.getCount(inner.buf)) {
-                inner(inner.io).store(inner.buf, innerIdx, leaf.io, leaf.buf, cnt - 1);
+                // Update inner key with the new biggest key of left subtree.
+                inner.io.store(inner.buf, innerIdx, leaf.io, leaf.buf, cnt - 1);
                 leaf.io.setRemoveId(leaf.buf, globalRmvId.get());
             }
             else {
+                // If after leaf merge parent have lost inner key, we don't need to update it anymore.
                 assert innerIdx == inner.io.getCount(inner.buf);
                 assert inner(inner.io).getLeft(inner.buf, innerIdx) == leaf.page.id();
             }
-
-            // We can't merge the whole branch up here because of locking rules.
-            // Here we've already locked the whole branch from the bottom to the top.
         }
 
         /**
-         * @param lvl Level.
-         * @return {@code true} If merged, {@code false} if not (because of insufficient space).
+         * @param prnt Parent for merge.
+         * @return {@code true} If merged, {@code false} if not (because of insufficient space or empty parent).
          * @throws IgniteCheckedException If failed.
          */
-        private boolean merge(int lvl, boolean releaseMerged) throws IgniteCheckedException {
-            assert tail.lvl > lvl;
-
-            Tail<L> prnt = getTail(lvl + 1);
-
+        private boolean merge(Tail<L> prnt) throws IgniteCheckedException {
             if (prnt.io.getCount(prnt.buf) == 0)
                 return false; // Parent is an empty routing page, child forward page will have another parent.
 
-            Tail<L> cur = getTail(lvl);
-            Tail<L> back = cur.sibling;
+            Tail<L> right = prnt.down;
+            Tail<L> left = right.sibling;
 
-            assert prnt.down == cur : prnt.down;
-            assert back != null: "we must have a partner to merge with";
+            assert right.type == Tail.EXACT;
+            assert left != null: "we must have a partner to merge with";
 
-            if (back.type != Tail.BACK) { // Flip if it was actually FORWARD but not BACK.
-                assert back.type == Tail.FORWARD: back.type;
+            if (left.type != Tail.BACK) { // Flip if it was actually FORWARD but not BACK.
+                assert left.type == Tail.FORWARD: left.type;
 
-                back = cur; // Current goes back.
-                cur = cur.sibling; // Forward becomes current.
+                left = right;
+                right = right.sibling;
             }
 
-            assert cur.io == back.io: "must always be the same"; // Otherwise can be not compatible.
+            assert right.io == left.io: "must always be the same"; // Otherwise can be not compatible.
 
-            if (!mergePages(prnt, back, cur))
+            if (!mergePages(prnt, left, right))
                 return false;
 
-            // BACK becomes EXACT.
-            if (back.type == Tail.BACK) {
-                assert back.sibling == null;
-
-                back.down = cur.down;
-                back.type = Tail.EXACT;
-                prnt.down = back;
-            }
-            else { // back is already EXACT
-                assert back.type == Tail.EXACT: back.type;
+            // left from BACK becomes EXACT.
+            if (left.type == Tail.BACK) {
+                assert left.sibling == null;
 
-                back.sibling = null;
+                left.down = right.down;
+                left.type = Tail.EXACT;
+                prnt.down = left;
             }
+            else { // left is already EXACT.
+                assert left.type == Tail.EXACT: left.type;
 
-            // Always unlock and release current because we made it invisible for further code.
-            writeUnlockAndClose(cur.page);
-
-            if (releaseMerged) {
-                prnt.down = null;
-
-                writeUnlockAndClose(back.page);
+                left.sibling = null;
             }
 
             return true;
@@ -2248,7 +2151,7 @@ public abstract class BPlusTree<L, T extends L> {
          */
         private Tail(Page page, ByteBuffer buf, BPlusIO<L> io, byte type, int lvl, int idx) {
             assert type == BACK || type == EXACT || type == FORWARD: type;
-            assert idx == Integer.MIN_VALUE || (idx >= 0 && idx <= Short.MAX_VALUE): idx ;
+            assert idx == -1 || (idx >= 0 && idx <= Short.MAX_VALUE): idx ;
             assert lvl >= 0 && lvl <= Byte.MAX_VALUE: lvl;
             assert page != null;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/74dfb8f9/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index e3919f1..6dcb237 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -61,6 +61,15 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     private static final int CACHE_ID = 100500;
 
     /** */
+    private static int MAX_COUNT = 0;
+
+    /** */
+    private static int PUT_INC = 1;
+
+    /** */
+    private static int RMV_INC = 1;
+
+    /** */
     private PageMemory pageMem;
 
     /** */
@@ -87,8 +96,62 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     }
 
     /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
+    @Override protected void afterTest() throws Exception {
         pageMem.stop();
+
+        MAX_COUNT = 0;
+        PUT_INC = 1;
+        RMV_INC = -1;
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testSingle1() throws IgniteCheckedException {
+        MAX_COUNT = 1;
+        PUT_INC = -1;
+        RMV_INC = -1;
+
+        doTestPutRemoveForwardBackward(true);
+    }
+
+    /**
+     * @param canGetRow Can get row from inner page.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void doTestPutRemoveForwardBackward(boolean canGetRow) throws IgniteCheckedException {
+        TestTree tree = createTestTree(canGetRow);
+
+        long cnt = 10;
+
+        for (long x = PUT_INC > 0 ? 0 : cnt - 1; x >= 0 && x < cnt; x += PUT_INC) {
+            assertNull(tree.findOne(x));
+
+            tree.put(x);
+
+            assertEquals(x, tree.findOne(x).longValue());
+        }
+
+        X.println(tree.printTree());
+
+        assertNull(tree.findOne(-1L));
+
+        for (long x = 0; x < cnt; x++)
+            assertEquals(x, tree.findOne(x).longValue());
+
+        assertNull(tree.findOne(cnt));
+
+        for (long x = RMV_INC > 0 ? 0 : cnt - 1; x >= 0 && x < cnt; x += RMV_INC) {
+            X.println(" -- " + x);
+
+            assertEquals(x, tree.remove(x).longValue());
+
+            assertNull(tree.findOne(x));
+
+            X.println(tree.printTree());
+        }
+
+        assertFalse(tree.find(null, null).next());
     }
 
     /**
@@ -438,6 +501,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     }
 
     /**
+     * TODO refactor to use integer in inner page
      * Long inner.
      */
     private static final class LongInnerIO extends BPlusInnerIO<Long> {
@@ -454,6 +518,14 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
         }
 
         /** {@inheritDoc} */
+        @Override public int getMaxCount(ByteBuffer buf) {
+            if (MAX_COUNT != 0)
+                return MAX_COUNT;
+
+            return super.getMaxCount(buf);
+        }
+
+        /** {@inheritDoc} */
         @Override public void store(ByteBuffer dst, int dstIdx, BPlusIO<Long> srcIo, ByteBuffer src, int srcIdx)
             throws IgniteCheckedException {
             store(dst, dstIdx, srcIo.getLookupRow(null, src, srcIdx));
@@ -485,11 +557,26 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
         }
 
         /** {@inheritDoc} */
+        @Override public int getMaxCount(ByteBuffer buf) {
+            if (MAX_COUNT != 0)
+                return MAX_COUNT;
+
+            return super.getMaxCount(buf);
+        }
+
+        /** {@inheritDoc} */
         @Override public void store(ByteBuffer buf, int idx, Long row) {
             buf.putLong(offset(idx), row);
         }
 
         /** {@inheritDoc} */
+        @Override public void store(ByteBuffer dst, int dstIdx, BPlusIO<Long> srcIo, ByteBuffer src, int srcIdx) {
+            assert srcIo == this;
+
+            dst.putLong(offset(dstIdx), src.getLong(offset(srcIdx)));
+        }
+
+        /** {@inheritDoc} */
         @Override public Long getLookupRow(BPlusTree<Long,?> tree, ByteBuffer buf, int idx)
             throws IgniteCheckedException {
             return buf.getLong(offset(idx));


[05/50] ignite git commit: ignite-db - getForWrite is not reentrant

Posted by sb...@apache.org.
ignite-db - getForWrite is not reentrant


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/24a345a0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/24a345a0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/24a345a0

Branch: refs/heads/ignite-db-x-10884
Commit: 24a345a00c23a54963ea2fe7c9388ecb69a2ff1a
Parents: 1ec6712
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 20 23:18:39 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 20 23:18:39 2016 +0300

----------------------------------------------------------------------
 .../apache/ignite/internal/pagemem/impl/PageImpl.java   | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/24a345a0/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
index 77f096d..9f42d93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageImpl.java
@@ -151,8 +151,8 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
 
     /** {@inheritDoc} */
     @Override public ByteBuffer getForInitialWrite() {
-        assert getExclusiveOwnerThread() == null;
-        assert getState() == 0;
+        assert getExclusiveOwnerThread() == null: fullId();
+        assert getState() == 0: fullId();
 
         markDirty();
 
@@ -161,13 +161,11 @@ class PageImpl extends AbstractQueuedSynchronizer implements Page {
 
     /** {@inheritDoc} */
     @Override public ByteBuffer getForWrite() {
-        Thread th = Thread.currentThread();
+        acquire(1); // This call is not reentrant.
 
-        if (getExclusiveOwnerThread() != th) {
-            acquire(1);
+        assert getExclusiveOwnerThread() == null: fullId();
 
-            setExclusiveOwnerThread(th);
-        }
+        setExclusiveOwnerThread(Thread.currentThread());
 
         return reset(buf);
     }


[22/50] ignite git commit: Added test.

Posted by sb...@apache.org.
Added test.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bb796880
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bb796880
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bb796880

Branch: refs/heads/ignite-db-x-10884
Commit: bb7968801d2cd60203022adbeec3ca4d92110937
Parents: fc2d77d
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Apr 25 13:26:56 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Apr 25 13:26:56 2016 -0700

----------------------------------------------------------------------
 .../IgniteDbMultiNodePutGetSelfTest.java        |  5 ++
 .../IgniteDbSingleNodePutGetSelfTest.java       | 66 ++++++++++++++++++++
 2 files changed, 71 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/bb796880/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMultiNodePutGetSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMultiNodePutGetSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMultiNodePutGetSelfTest.java
index bd9fc26..1ead18f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMultiNodePutGetSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMultiNodePutGetSelfTest.java
@@ -17,4 +17,9 @@ public class IgniteDbMultiNodePutGetSelfTest extends IgniteDbSingleNodePutGetSel
     @Override protected int gridCount() {
         return 3;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testMultithreadedPut() throws Exception {
+        super.testMultithreadedPut();
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/bb796880/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
index 7570f80..7b7d791 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodePutGetSelfTest.java
@@ -15,6 +15,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -270,7 +271,69 @@ public class IgniteDbSingleNodePutGetSelfTest extends GridCommonAbstractTest {
         assertTrue(plan, plan.contains("iVal_idx"));
     }
 
+    /**
+     * @throws Exception if failed.
+     */
+    public void testMultithreadedPut() throws Exception {
+        IgniteEx ig = grid(0);
+
+        final IgniteCache<Integer, DbValue> cache = ig.cache(null);
+
+        X.println("Put start");
+
+        int cnt = 20_000;
+
+        try (IgniteDataStreamer<Integer, DbValue> st = ig.dataStreamer(null)) {
+            st.allowOverwrite(true);
+
+            for (int i = 0; i < cnt; i++) {
+                DbValue v0 = new DbValue(i, "test-value", i);
+
+                st.addData(i, v0);
+            }
+        }
+
+        X.println("Get start");
+
+        for (int i = 0; i < cnt; i++) {
+            DbValue v0 = new DbValue(i, "test-value", i);
+
+            assertEquals(v0, cache.get(i));
+        }
+
+        awaitPartitionMapExchange();
+
+        X.println("Query start");
+
+        assertEquals(cnt, cache.query(new SqlFieldsQuery("select null from dbvalue")).getAll().size());
+
+        int limit = 500;
+
+        List<List<?>> res = cache.query(new SqlFieldsQuery("select ival, _val from dbvalue where ival < ? order by ival")
+            .setArgs(limit)).getAll();
+
+        assertEquals(limit, res.size());
+
+        for (int i = 0; i < limit; i++) {
+            List<?> row = res.get(i);
+
+            assertEquals(2, row.size());
+            assertEquals(i, row.get(0));
+
+            assertEquals(new DbValue(i, "test-value", i), row.get(1));
+        }
 
+        assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
+        assertEquals(2000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 7000"))
+            .getAll().size());
+
+        String plan = cache.query(new SqlFieldsQuery(
+            "explain select lval from dbvalue where ival >= 5000 and ival < 7000")).getAll().get(0).get(0).toString();
+
+        assertTrue(plan, plan.contains("iVal_idx"));
+
+        cache.clear();
+    }
 
     /**
      * @throws Exception if failed.
@@ -435,6 +498,9 @@ public class IgniteDbSingleNodePutGetSelfTest extends GridCommonAbstractTest {
         }
     }
 
+    /**
+     * @throws Exception if failed.
+     */
     public void testPutGetRemoveMultipleForward() throws Exception {
         IgniteEx ig = grid(0);
 


[11/50] ignite git commit: ignite-db - tails wip

Posted by sb...@apache.org.
ignite-db - tails wip


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ff88620d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ff88620d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ff88620d

Branch: refs/heads/ignite-db-x-10884
Commit: ff88620d571dea58cbc6403ccbe64bd9f2f55632
Parents: e7d7e2c
Author: S.Vladykin <sv...@gridgain.com>
Authored: Fri Apr 22 17:41:23 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Fri Apr 22 17:41:23 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 95 +++++++++++++++-----
 1 file changed, 74 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ff88620d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 1e72677..a822923 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -318,13 +318,17 @@ public abstract class BPlusTree<L, T extends L> {
 
             r.removed = getRow(io, buf, idx);
 
-            r.doRemove(io, leaf, buf, cnt, idx, lvl, false);
+            r.doRemove(io, leaf, buf, cnt, idx, 0, false);
 
             // We may need to replace inner key or want to merge this leaf with sibling after the remove -> keep lock.
             if (r.needReplaceInner == TRUE ||
                 // We need to make sure that we have back or forward to be able to merge.
                 ((r.fwdId != 0 || r.backId != 0) && mayMerge(cnt - 1, io.getMaxCount(buf)))) {
-                r.addTail(leaf, buf, io, 0, false, Integer.MIN_VALUE);
+                r.addTail(leaf, buf, io, 0, false, Tail.PRIMARY, Integer.MIN_VALUE);
+
+                if (r.fwdId != 0 && r.backId == 0) // If we have backId then we'll lock back page, nothing to do here.
+                    r.lockForward(0);
+
 
                 if (r.needReplaceInner == FALSE)
                     r.needMerge = TRUE;
@@ -368,31 +372,42 @@ public abstract class BPlusTree<L, T extends L> {
             int res = r.doLockTail(lvl);
 
             if (res == Remove.FOUND)
-                r.addTail(back, buf, io, lvl, true, Integer.MIN_VALUE);
+                r.addTail(back, buf, io, lvl, true, false, Integer.MIN_VALUE);
 
             return res;
         }
     };
 
     /** */
+    private final PageHandler<Remove> lockTailForward = new GetPageHandler<Remove>() {
+        @Override protected int run0(Page page, ByteBuffer buf, BPlusIO<L> io, Remove r, int lvl)
+            throws IgniteCheckedException {
+
+            r.addTail(page, buf, io, lvl, false, false, Integer.MIN_VALUE);
+
+            return Remove.FOUND;
+        }
+    };
+
+    /** */
     private final PageHandler<Remove> lockTail = new GetPageHandler<Remove>() {
         @Override public int run0(Page page, ByteBuffer buf, BPlusIO<L> io, Remove r, int lvl)
             throws IgniteCheckedException {
+            assert lvl > 0: lvl; // We are not at the bottom.
+
             int cnt = io.getCount(buf);
             int idx = findInsertionPoint(io, buf, cnt, r.row);
 
             boolean found = idx >= 0;
 
             if (found) {
-                if (lvl != 0 && r.needReplaceInner == TRUE) {
-                    assert idx <= Short.MAX_VALUE : idx;
+                // We could not miss the inner value on down move because of `triangle` invariant, thus it must be TRUE.
+                assert r.needReplaceInner == TRUE: r.needReplaceInner;
+                assert idx <= Short.MAX_VALUE : idx;
 
-                    r.innerIdx = (short)idx;
+                r.innerIdx = (short)idx;
 
-                    r.needReplaceInner = READY;
-                }
-                else
-                    assert r.needMerge == TRUE: r.needMerge;
+                r.needReplaceInner = READY;
             }
             else {
                 idx = -idx - 1;
@@ -409,7 +424,14 @@ public abstract class BPlusTree<L, T extends L> {
                 return Remove.RETRY;
             }
 
-            r.addTail(page, buf, io, lvl, false, idx);
+            // We don't have a back page, need to lock our forward and become a back for it.
+            // If found then we are on inner replacement page, it will be a top parent, no need to lock forward.
+            boolean back = !found && r.fwdId != 0 && r.backId == 0;
+
+            if (back)
+                r.lockForward(lvl);
+
+            r.addTail(page, buf, io, lvl, back, true, idx);
 
             if (r.needMerge == TRUE) {
                 assert r.needReplaceInner == FALSE;
@@ -1614,7 +1636,7 @@ public abstract class BPlusTree<L, T extends L> {
         Tail<L> tail;
 
         /** */
-        List<FullPageId> emptyPages;
+        List<FullPageId> emptyPages; // TODO May be use Object for single empty page
 
         /** */
         byte needReplaceInner = FALSE;
@@ -1730,6 +1752,7 @@ public abstract class BPlusTree<L, T extends L> {
          * @throws IgniteCheckedException If failed.
          */
         private int removeFromLeaf(Page leaf, long backId, long fwdId) throws IgniteCheckedException {
+            // Init parameters.
             this.pageId = leaf.id();
             this.page = leaf;
             this.backId = backId;
@@ -1789,7 +1812,7 @@ public abstract class BPlusTree<L, T extends L> {
             this.fwdId = fwdId;
             this.backId = backId;
 
-            if (backId == 0) // Back page ID is provided only when last move was to the right.
+            if (backId == 0) // Back page ID is provided only when the last move was to the right.
                 return doLockTail(lvl);
 
             Page back = page(backId);
@@ -1804,6 +1827,20 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
+         * @param lvl Level.
+         * @throws IgniteCheckedException If failed.
+         */
+        private void lockForward(int lvl) throws IgniteCheckedException {
+            assert needReplaceInner == TRUE: needReplaceInner;
+            assert fwdId != 0;
+            assert backId == 0;
+
+            int res = writePage(page(fwdId), lockTailForward, this, lvl, Remove.RETRY);
+
+            assert res == Remove.FOUND;
+        }
+
+        /**
          * @param io IO.
          * @param buf Buffer.
          * @param cnt Count.
@@ -2141,10 +2178,13 @@ public abstract class BPlusTree<L, T extends L> {
          * @param io IO.
          * @param lvl Level.
          * @param back If the given page is back page for the current tail, otherwise it must be an upper level page.
-         * @param idx Insertion index.
+         * @param idx Insertion index or negative flag describing if the page is primary in this tail branch.
          */
-        private void addTail(Page page, ByteBuffer buf, BPlusIO<L> io, int lvl, boolean back, int idx) {
-            Tail<L> t = new Tail<>(page, buf, io, lvl, idx);
+        private void addTail(Page page, ByteBuffer buf, BPlusIO<L> io,
+            int lvl, boolean back, byte primary, int idx) {
+            assert primary == Tail.PRIMARY || primary == Tail.COMPLIMENTARY: primary;
+
+            Tail<L> t = new Tail<>(page, buf, io, primary == Tail.PRIMARY, lvl, idx);
 
             if (back) {
                 assert tail != null;
@@ -2194,6 +2234,12 @@ public abstract class BPlusTree<L, T extends L> {
      */
     private static class Tail<L> {
         /** */
+        static final byte PRIMARY = 1;
+
+        /** */
+        static final byte COMPLIMENTARY = 2;
+
+        /** */
         final Page page;
 
         /** */
@@ -2203,10 +2249,13 @@ public abstract class BPlusTree<L, T extends L> {
         final BPlusIO<L> io;
 
         /** */
-        final int lvl;
+        final boolean primary;
+
+        /** */
+        final byte lvl;
 
         /** */
-        final int idx;
+        final short idx;
 
         /** */
         Tail<L> fwd;
@@ -2218,17 +2267,21 @@ public abstract class BPlusTree<L, T extends L> {
          * @param page Write locked page.
          * @param buf Buffer.
          * @param io IO.
+         * @param primary Primary.
          * @param lvl Level.
          * @param idx Insertion index.
          */
-        private Tail(Page page, ByteBuffer buf, BPlusIO<L> io, int lvl, int idx) {
+        private Tail(Page page, ByteBuffer buf, BPlusIO<L> io, boolean primary, int lvl, int idx) {
+            assert idx == Integer.MIN_VALUE || (idx >= 0 && idx <= Short.MAX_VALUE): idx ;
+            assert lvl >= 0 && lvl <= Byte.MAX_VALUE: lvl;
             assert page != null;
 
             this.page = page;
             this.buf = buf;
             this.io = io;
-            this.lvl = lvl;
-            this.idx = idx;
+            this.primary = primary;
+            this.lvl = (byte)lvl;
+            this.idx = (short)idx;
         }
     }
 


[42/50] ignite git commit: ignite-db - fixes

Posted by sb...@apache.org.
ignite-db - fixes


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/50e452c9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/50e452c9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/50e452c9

Branch: refs/heads/ignite-db-x-10884
Commit: 50e452c9296a3057dcf23d5c062f65d964d9ee23
Parents: ca839c8
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 10:03:02 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 10:03:02 2016 +0300

----------------------------------------------------------------------
 .../ignite/internal/processors/query/h2/database/io/H2InnerIO.java | 2 --
 .../ignite/internal/processors/query/h2/database/io/H2LeafIO.java  | 2 --
 2 files changed, 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/50e452c9/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
index d93cf0e..74be639 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
@@ -69,8 +69,6 @@ public class H2InnerIO extends BPlusInnerIO<SearchRow> implements H2RowLinkIO {
 
     /** {@inheritDoc} */
     @Override public long getLink(ByteBuffer buf, int idx) {
-        assert idx < getCount(buf): idx;
-
         return buf.getLong(offset(idx));
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/50e452c9/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
index c2c915a..68c8766 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
@@ -69,8 +69,6 @@ public class H2LeafIO extends BPlusLeafIO<SearchRow> implements H2RowLinkIO {
 
     /** {@inheritDoc} */
     @Override public long getLink(ByteBuffer buf, int idx) {
-        assert idx < getCount(buf): idx;
-
         return buf.getLong(offset(idx));
     }
 


[06/50] ignite git commit: ignite-db - minor

Posted by sb...@apache.org.
ignite-db - minor


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d7e35ee4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d7e35ee4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d7e35ee4

Branch: refs/heads/ignite-db-x-10884
Commit: d7e35ee4d025dda3e71b9178c36093f82e057ce0
Parents: 24a345a
Author: S.Vladykin <sv...@gridgain.com>
Authored: Thu Apr 21 00:35:50 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Thu Apr 21 00:35:50 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 22 +++++++-------------
 1 file changed, 7 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/d7e35ee4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 98fd8eb..61c49d7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -403,7 +403,7 @@ public abstract class BPlusTree<L, T extends L> {
             }
 
             // Check that we have a correct view of the world.
-            if (lvl != 0 && inner(io).getLeft(buf, idx) != r.nonBackTailPage().id()) {
+            if (lvl != 0 && inner(io).getLeft(buf, idx) != r.getTail(lvl - 1, false).page.id()) {
                 assert !found;
 
                 return Remove.RETRY;
@@ -806,7 +806,7 @@ public abstract class BPlusTree<L, T extends L> {
     public final T removeCeil(L row) throws IgniteCheckedException {
         assert row != null;
 
-        return remove(row, true);
+        return doRemove(row, true);
     }
 
     /**
@@ -817,7 +817,7 @@ public abstract class BPlusTree<L, T extends L> {
     public final T remove(L row) throws IgniteCheckedException {
         assert row != null;
 
-        return remove(row, false);
+        return doRemove(row, false);
     }
 
     /**
@@ -826,7 +826,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Removed row.
      * @throws IgniteCheckedException If failed.
      */
-    public final T remove(L row, boolean ceil) throws IgniteCheckedException {
+    public final T doRemove(L row, boolean ceil) throws IgniteCheckedException {
         Remove r = new Remove(row, ceil);
 
         try {
@@ -2157,21 +2157,13 @@ public abstract class BPlusTree<L, T extends L> {
         }
 
         /**
-         * @return Non-back tail page.
-         */
-        private Page nonBackTailPage() {
-            assert tail != null;
-
-            return tail.fwd == null ? tail.page : tail.fwd.page;
-        }
-
-        /**
          * @param lvl Level.
          * @param back Back page.
          * @return Tail.
          */
         private Tail<L> getTail(int lvl, boolean back) {
-            assert lvl <= tail.lvl: "level is too high";
+            assert tail != null;
+            assert lvl >= 0 && lvl <= tail.lvl: lvl;
 
             Tail<L> t = tail;
 
@@ -2181,7 +2173,7 @@ public abstract class BPlusTree<L, T extends L> {
                 else
                     t = t.down;
 
-                assert t != null : "level is too low";
+                assert t != null: lvl;
             }
 
             if (back)


[24/50] ignite git commit: Full page ID to ignore partition ID for index pages.

Posted by sb...@apache.org.
Full page ID to ignore partition ID for index pages.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ad5aacfa
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ad5aacfa
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ad5aacfa

Branch: refs/heads/ignite-db-x-10884
Commit: ad5aacfae6c56ea5c3e543d200bf4e20771047b8
Parents: 823fabd
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Apr 25 17:20:30 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Apr 25 17:20:30 2016 -0700

----------------------------------------------------------------------
 .../ignite/internal/pagemem/FullPageId.java     | 21 +++--
 .../ignite/internal/pagemem/PageIdUtils.java    | 86 +++++++++++---------
 .../internal/pagemem/impl/FullPageIdTable.java  | 21 ++---
 .../pagemem/impl/PageIdUtilsSelfTest.java       | 39 +++++++++
 4 files changed, 112 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/ad5aacfa/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
index 22a01c4..0247aec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
@@ -19,23 +19,30 @@ package org.apache.ignite.internal.pagemem;
 
 import org.apache.ignite.internal.util.typedef.internal.SB;
 
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
+
 /**
  *
  */
 public class FullPageId {
     /** */
-    private long pageId;
+    private final long pageId;
+
+    /** */
+    private final long effectivePageId;
 
     /** */
-    private int cacheId;
+    private final int cacheId;
 
     /**
      * @param pageId Page ID.
      * @param cacheId Cache ID.
      */
     public FullPageId(long pageId, int cacheId) {
-        this.pageId = pageId;
+        this.pageId = PageIdUtils.pageId(pageId);
         this.cacheId = cacheId;
+
+        effectivePageId = PageIdUtils.flag(pageId) == FLAG_IDX ? PageIdUtils.effectiveIndexPageId(pageId) : this.pageId;
     }
 
     /**
@@ -62,12 +69,12 @@ public class FullPageId {
 
         FullPageId that = (FullPageId)o;
 
-        return pageId == that.pageId && cacheId == that.cacheId;
+        return effectivePageId == that.effectivePageId && cacheId == that.cacheId;
     }
 
     /** {@inheritDoc} */
     @Override public int hashCode() {
-        int result = (int)(pageId ^ (pageId >>> 32));
+        int result = (int)(effectivePageId ^ (effectivePageId >>> 32));
 
         result = 31 * result + cacheId;
 
@@ -76,6 +83,8 @@ public class FullPageId {
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return new SB("FullPageId [pageId=").appendHex(pageId).a(", cacheId=").a(cacheId).a(']').toString();
+        return new SB("FullPageId [pageId=").appendHex(pageId)
+            .a(", effectivePageId=").appendHex(effectivePageId)
+            .a(", cacheId=").a(cacheId).a(']').toString();
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/ad5aacfa/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
index 4493b1a..3a14c5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
@@ -18,47 +18,55 @@
 package org.apache.ignite.internal.pagemem;
 
 /**
- * TODO describe the bits structure.
+ * Utility class for page ID parts manipulation.
+ *
+ * @see FullPageId
  */
 public final class PageIdUtils {
     /** */
-    private static final long FILE_ID_MASK  = 0x000FFFFFC0000000L;
+    private static final int PAGE_IDX_SIZE = 30;
 
     /** */
-    private static final long PAGE_NUM_MASK = 0x000000003FFFFFFFL;
+    private static final int FILE_ID_SIZE = 22;
 
     /** */
-    private static final int OFFSET_SHIFTED_MASK = 0xFFF;
+    private static final int PART_ID_SIZE = 14;
 
     /** */
-    private static final int FILE_ID_SIZE = Long.bitCount(FILE_ID_MASK);
+    private static final int FLAG_SIZE = 3;
 
     /** */
-    private static final int PART_ID_SIZE = 14;
+    private static final int OFFSET_SIZE = 12;
 
     /** */
-    private static final int PART_ID_MASK = ~(-1 << PART_ID_SIZE);
+    private static final long PAGE_IDX_MASK = ~(-1 << PAGE_IDX_SIZE);
 
     /** */
-    private static final int FLAG_SIZE = 3;
+    private static final long FILE_ID_MASK = ~(-1 << FILE_ID_SIZE);
+
+    /** */
+    private static final long OFFSET_MASK = ~(-1 << OFFSET_SIZE);
+
+    /** */
+    private static final int PART_ID_MASK = ~(-1 << PART_ID_SIZE);
 
     /** */
     private static final int FLAG_MASK = ~(-1 << FLAG_SIZE);
 
     /** */
-    private static final int PAGE_NUM_SIZE = Long.bitCount(PAGE_NUM_MASK);
+    private static final long EFFECTIVE_INDEX_PAGE_ID_MASK = ((long)FLAG_MASK << (PAGE_IDX_SIZE + PART_ID_SIZE)) | PAGE_IDX_MASK;
 
     /** Maximum page number. */
-    public static final int MAX_PAGE_NUM = (1 << PAGE_NUM_SIZE) - 1;
+    public static final int MAX_PAGE_NUM = (1 << PAGE_IDX_SIZE) - 1;
+
+    /** Maximum page number. */
+    public static final int MAX_PART_ID = (1 << PART_ID_SIZE) - 1;
 
     /** Maximum file ID. */
     public static final int MAX_FILE_ID = (1 << FILE_ID_SIZE) - 1;
 
     /** Maximum offset in dwords. */
-    public static final int MAX_OFFSET_DWORDS = OFFSET_SHIFTED_MASK;
-
-    /** Maximum part number. */
-    public static final int MAX_PART_ID = (1 << PART_ID_SIZE) - 1;
+    public static final int MAX_OFFSET_DWORDS = (int)OFFSET_MASK;
 
     /**
      *
@@ -77,10 +85,10 @@ public final class PageIdUtils {
      */
     public static long linkFromBytesOffset(long pageId, int bytesOffset) {
         assert (bytesOffset & 0x7) == 0;
-        assert (pageId >> (PAGE_NUM_SIZE + FILE_ID_SIZE)) == 0;
+        assert (pageId >> (PAGE_IDX_SIZE + FILE_ID_SIZE)) == 0;
 
-        // (bytesOffset >> 3) << PAGE_NUM_SIZE
-        return pageId | (((long)bytesOffset) << (FILE_ID_SIZE + PAGE_NUM_SIZE - 3));
+        // (bytesOffset >> 3) << PAGE_IDX_SIZE
+        return pageId | (((long)bytesOffset) << (FILE_ID_SIZE + PAGE_IDX_SIZE - 3));
     }
 
     /**
@@ -91,9 +99,9 @@ public final class PageIdUtils {
      * @return Page link.
      */
     public static long linkFromDwordOffset(long pageId, int dwordOffset) {
-        assert (pageId >> (PAGE_NUM_SIZE + FILE_ID_SIZE)) == 0;
+        assert (pageId >> (PAGE_IDX_SIZE + FILE_ID_SIZE)) == 0;
 
-        return pageId | (((long)dwordOffset) << (PAGE_NUM_SIZE + FILE_ID_SIZE));
+        return pageId | (((long)dwordOffset) << (PAGE_IDX_SIZE + FILE_ID_SIZE));
     }
 
     /**
@@ -104,9 +112,14 @@ public final class PageIdUtils {
      * @return Page ID.
      */
     public static long pageId(int fileId, long pageIdx) {
-        assert (pageIdx & ~PAGE_NUM_MASK) == 0;
+        assert (pageIdx & ~PAGE_IDX_MASK) == 0;
+
+        long pageId = 0;
+
+        pageId = (pageId << FILE_ID_SIZE) | (fileId & FILE_ID_MASK);
+        pageId = (pageId << PAGE_IDX_SIZE) | (pageIdx & PAGE_IDX_MASK);
 
-        return (( ((long)fileId) << PAGE_NUM_SIZE) & FILE_ID_MASK ) | pageIdx;
+        return pageId;
     }
 
     /**
@@ -116,7 +129,7 @@ public final class PageIdUtils {
      * @return Page ID.
      */
     public static long pageIdx(long pageId) {
-        return pageId & PAGE_NUM_MASK;
+        return pageId & PAGE_IDX_MASK;
     }
 
     /**
@@ -126,7 +139,15 @@ public final class PageIdUtils {
      * @return Page ID.
      */
     public static long pageId(long link) {
-        return link & (FILE_ID_MASK | PAGE_NUM_MASK);
+        return link & ~(OFFSET_MASK << (FILE_ID_SIZE + PAGE_IDX_SIZE));
+    }
+
+    /**
+     * @param link Page link.
+     * @return Effective index page id.
+     */
+    public static long effectiveIndexPageId(long link) {
+        return link & EFFECTIVE_INDEX_PAGE_ID_MASK;
     }
 
     /**
@@ -136,7 +157,7 @@ public final class PageIdUtils {
      * @return File ID.
      */
     public static int fileId(long linkOrPageId) {
-        return (int)((linkOrPageId & FILE_ID_MASK) >> PAGE_NUM_SIZE);
+        return (int)((linkOrPageId >> PAGE_IDX_SIZE) & FILE_ID_MASK);
     }
 
     /**
@@ -146,7 +167,7 @@ public final class PageIdUtils {
      * @return Offset within the page in bytes.
      */
     public static int bytesOffset(long link) {
-        return (int)((link >> (PAGE_NUM_SIZE + FILE_ID_SIZE)) & OFFSET_SHIFTED_MASK) << 3;
+        return (int)((link >> (PAGE_IDX_SIZE + FILE_ID_SIZE)) & OFFSET_MASK) << 3;
     }
 
     /**
@@ -156,16 +177,7 @@ public final class PageIdUtils {
      * @return Offset in 8-byte words.
      */
     public static int dwordsOffset(long link) {
-        return (int)(link >> (PAGE_NUM_SIZE + FILE_ID_SIZE)) & OFFSET_SHIFTED_MASK;
-    }
-
-    /**
-     * @param cacheId Cache ID.
-     * @param partId Partition ID.
-     * @return File ID constructed from the given cache ID and partition ID.
-     */
-    public static int fileId(int cacheId, int partId) {
-        return (cacheId & 0xFF << 14) | partId;
+        return (int)((link >> (PAGE_IDX_SIZE + FILE_ID_SIZE)) & OFFSET_MASK);
     }
 
     /**
@@ -183,10 +195,10 @@ public final class PageIdUtils {
     }
 
     public static byte flag(long pageId) {
-        return (byte) (( pageId >>> (PART_ID_SIZE + PAGE_NUM_SIZE) ) & FLAG_MASK);
+        return (byte) (( pageId >>> (PART_ID_SIZE + PAGE_IDX_SIZE) ) & FLAG_MASK);
     }
 
     public static int partId(long pageId) {
-        return (int) ((pageId >>> PAGE_NUM_SIZE) & PART_ID_MASK);
+        return (int) ((pageId >>> PAGE_IDX_SIZE) & PART_ID_MASK);
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/ad5aacfa/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
index e05d64c..9c4ce66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/FullPageIdTable.java
@@ -46,13 +46,10 @@ public class FullPageIdTable {
     private static final int EMPTY_CACHE_ID = EMPTY_FULL_PAGE_ID.cacheId();
 
     /** */
-    public static final FullPageId REMOVED_FULL_PAGE_ID = new FullPageId(0x8000000000000000L, 0);
+    private static final long REMOVED_PAGE_ID = 0x8000000000000000L;
 
     /** */
-    private static final long REMOVED_PAGE_ID = REMOVED_FULL_PAGE_ID.pageId();
-
-    /** */
-    private static final int REMOVED_CACHE_ID = REMOVED_FULL_PAGE_ID.cacheId();
+    private static final int REMOVED_CACHE_ID = 0;
 
     /** */
     private static final int EQUAL = 0;
@@ -179,7 +176,7 @@ public class FullPageIdTable {
             int res = testKeyAt(index, key);
 
             if (res == EMPTY) {
-                setKeyAt(index, key);
+                setKeyAt(index, key.pageId(), key.cacheId());
 
                 incrementSize();
 
@@ -237,7 +234,7 @@ public class FullPageIdTable {
             long res = testKeyAt(index, key);
 
             if (res == EQUAL) {
-                setKeyAt(index, REMOVED_FULL_PAGE_ID);
+                setKeyAt(index, REMOVED_PAGE_ID, REMOVED_CACHE_ID);
 
                 decrementSize();
 
@@ -283,20 +280,20 @@ public class FullPageIdTable {
      */
     private boolean assertKey(FullPageId fullId) {
         assert !F.eq(fullId, EMPTY_FULL_PAGE_ID) : "fullId != EMPTY";
-        assert !F.eq(fullId, REMOVED_FULL_PAGE_ID) : "fullId != REMOVED";
 
         return true;
     }
 
     /**
      * @param index Entry index.
-     * @param val Value to write.
+     * @param pageId Page ID to write.
+     * @param cacheId Cache ID to write.
      */
-    private void setKeyAt(int index, FullPageId val) {
+    private void setKeyAt(int index, long pageId, int cacheId) {
         long base = valPtr + 4 + (long)index * BYTES_PER_ENTRY;
 
-        mem.writeLong(base, val.pageId());
-        mem.writeLong(base + 8, val.cacheId());
+        mem.writeLong(base, pageId);
+        mem.writeLong(base + 8, cacheId);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/ad5aacfa/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageIdUtilsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageIdUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageIdUtilsSelfTest.java
index 1306204..77ab497 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageIdUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageIdUtilsSelfTest.java
@@ -10,7 +10,10 @@
 package org.apache.ignite.internal.pagemem.impl;
 
 import java.util.Random;
+
+import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -146,6 +149,42 @@ public class PageIdUtilsSelfTest extends GridCommonAbstractTest {
     }
 
     /**
+     * @throws Exception if failed.
+     */
+    public void testFullPageId() throws Exception {
+        Random rnd = new Random();
+
+        // Check that link is not included in FullPageId for data pages.
+        for (int i = 0; i < 50_000; i++) {
+            int offset = rnd.nextInt(PageIdUtils.MAX_OFFSET_DWORDS + 1);
+            int partId = rnd.nextInt(PageIdUtils.MAX_PART_ID + 1);
+            int pageNum = rnd.nextInt(PageIdUtils.MAX_PAGE_NUM + 1);
+
+            long pageId = PageIdUtils.pageId(partId, PageMemory.FLAG_DATA, pageNum);
+
+            long link1 = PageIdUtils.linkFromDwordOffset(pageId, offset);
+            long link2 = PageIdUtils.linkFromDwordOffset(pageId, 0);
+
+            assertEquals(new FullPageId(link1, 1), new FullPageId(link2, 1));
+        }
+
+        // Check that partition ID is not included in FullPageId for data pages.
+        for (int i = 0; i < 50_000; i++) {
+            int offset = rnd.nextInt(PageIdUtils.MAX_OFFSET_DWORDS + 1);
+            int partId = rnd.nextInt(PageIdUtils.MAX_PART_ID + 1);
+            int pageNum = rnd.nextInt(PageIdUtils.MAX_PAGE_NUM + 1);
+
+            long pageId1 = PageIdUtils.pageId(partId, PageMemory.FLAG_IDX, pageNum);
+            long pageId2 = PageIdUtils.pageId(1, PageMemory.FLAG_IDX, pageNum);
+
+            long link1 = PageIdUtils.linkFromDwordOffset(pageId1, offset);
+            long link2 = PageIdUtils.linkFromDwordOffset(pageId2, 0);
+
+            assertEquals(new FullPageId(link1, 1), new FullPageId(link2, 1));
+        }
+    }
+
+    /**
      * @throws Exception If failed.
      */
     public void testRandomIds() throws Exception {


[16/50] ignite git commit: ignite-db - minor

Posted by sb...@apache.org.
ignite-db - minor


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f444783b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f444783b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f444783b

Branch: refs/heads/ignite-db-x-10884
Commit: f444783b1d45a881dbfc2991152ce4ffffb256cc
Parents: d9a3a82
Author: S.Vladykin <sv...@gridgain.com>
Authored: Mon Apr 25 01:27:10 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Mon Apr 25 01:27:10 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 44 ++++++++++----------
 .../cache/database/tree/io/BPlusMetaIO.java     |  8 ++--
 2 files changed, 25 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/f444783b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index aa54554..039fc3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -445,7 +445,7 @@ public abstract class BPlusTree<L, T extends L> {
     };
 
     /** */
-    private final PageHandler<Long> updateLeftmost = new PageHandler<Long>() {
+    private final PageHandler<Long> updateFirst = new PageHandler<Long>() {
         @Override public int run(Page page, ByteBuffer buf, Long pageId, int lvl) throws IgniteCheckedException {
             assert pageId != null;
 
@@ -453,31 +453,29 @@ public abstract class BPlusTree<L, T extends L> {
 
             assert io.getLevelsCount(buf) > lvl;
 
-            io.setLeftmostPageId(buf, lvl, pageId);
-
             if (pageId == 0) {
-                assert lvl == io.getRootLevel(buf);
+                assert lvl == io.getRootLevel(buf); // Can drop only root.
 
                 io.setLevelsCount(buf, lvl); // Decrease tree height.
             }
+            else
+                io.setFirstPageId(buf, lvl, pageId);
 
             return TRUE;
         }
     };
 
     /** */
-    private final PageHandler<Long> updateRoot = new PageHandler<Long>() {
+    private final PageHandler<Long> newRoot = new PageHandler<Long>() {
         @Override public int run(Page page, ByteBuffer buf, Long rootPageId, int lvl) throws IgniteCheckedException {
+            assert rootPageId != null;
+
             BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(buf);
 
-            int cnt = io.getLevelsCount(buf);
+            assert lvl == io.getLevelsCount(buf);
 
-            if (rootPageId != null) {
-                io.setLevelsCount(buf, cnt + 1);
-                io.setLeftmostPageId(buf, cnt, rootPageId);
-            }
-            else
-                io.setLevelsCount(buf, cnt - 1);
+            io.setLevelsCount(buf, lvl + 1);
+            io.setFirstPageId(buf, lvl, rootPageId);
 
             return TRUE;
         }
@@ -526,7 +524,7 @@ public abstract class BPlusTree<L, T extends L> {
                 latestLeafIO().initNewPage(root.getForInitialWrite(), root.id());
 
                 io.setLevelsCount(buf, 1);
-                io.setLeftmostPageId(buf, 0, root.id());
+                io.setFirstPageId(buf, 0, root.id());
             }
         }
     }
@@ -550,7 +548,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @param lvl Level, if {@code 0} then it is a bottom level, if {@link Integer#MIN_VALUE}, then root.
      * @return Page ID.
      */
-    private long getLeftmostPageId(Page meta, int lvl) {
+    private long getFirstPageId(Page meta, int lvl) {
         ByteBuffer buf = meta.getForRead();
 
         try {
@@ -562,7 +560,7 @@ public abstract class BPlusTree<L, T extends L> {
             if (lvl >= io.getLevelsCount(buf))
                 return 0;
 
-            return io.getLeftmostPageId(buf, lvl);
+            return io.getFirstPageId(buf, lvl);
         }
         finally {
             meta.releaseRead();
@@ -579,7 +577,7 @@ public abstract class BPlusTree<L, T extends L> {
         long firstPageId;
 
         try (Page meta = page(metaPageId)) {
-            firstPageId = getLeftmostPageId(meta, 0); // Level 0 is always at the bottom.
+            firstPageId = getFirstPageId(meta, 0); // Level 0 is always at the bottom.
         }
 
         try (Page first = page(firstPageId)) {
@@ -717,7 +715,7 @@ public abstract class BPlusTree<L, T extends L> {
         long rootPageId;
 
         try (Page meta = page(metaPageId)) {
-            rootPageId = getLeftmostPageId(meta, Integer.MIN_VALUE);
+            rootPageId = getFirstPageId(meta, Integer.MIN_VALUE);
         }
         catch (IgniteCheckedException e) {
             throw new IllegalStateException(e);
@@ -1122,14 +1120,14 @@ public abstract class BPlusTree<L, T extends L> {
             // Do insert.
             int cnt = io.getCount(buf);
 
-            if (idx < cnt || (idx == cnt && !midShift)) {
+            if (idx < cnt || (idx == cnt && !midShift)) { // Insert into back page.
                 insertSimple(io, buf, row, idx, rightId);
 
                 // Fix leftmost child of forward page, because newly inserted row will go up.
                 if (idx == cnt && !io.isLeaf())
                     inner(io).setLeft(fwdBuf, 0, rightId);
             }
-            else
+            else // Insert into newly allocated forward page.
                 insertSimple(io, fwdBuf, row, idx - cnt, rightId);
 
             // Do move up.
@@ -1159,7 +1157,7 @@ public abstract class BPlusTree<L, T extends L> {
                     inner(io).setRight(newRootBuf, 0, fwd.id());
                 }
 
-                int res = writePage(meta, updateRoot, newRootId, lvl + 1, FALSE);
+                int res = writePage(meta, newRoot, newRootId, lvl + 1, FALSE);
 
                 assert res == TRUE : "failed to update meta page";
 
@@ -1387,7 +1385,7 @@ public abstract class BPlusTree<L, T extends L> {
                 BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(buf);
 
                 rootLvl = io.getRootLevel(buf);
-                rootId = io.getLeftmostPageId(buf, rootLvl);
+                rootId = io.getFirstPageId(buf, rootLvl);
             }
             finally {
                 meta.releaseRead();
@@ -1904,11 +1902,11 @@ public abstract class BPlusTree<L, T extends L> {
          */
         private void freePage(Page page, ByteBuffer buf, BPlusIO io, int lvl)
             throws IgniteCheckedException {
-            if (getLeftmostPageId(meta, lvl) == page.id()) {
+            if (getFirstPageId(meta, lvl) == page.id()) {
                 // This logic will handle root as well.
                 long fwdId = io.getForward(buf);
 
-                writePage(meta, updateLeftmost, fwdId, lvl, FALSE);
+                writePage(meta, updateFirst, fwdId, lvl, FALSE);
             }
 
             // Mark removed.

http://git-wip-us.apache.org/repos/asf/ignite/blob/f444783b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java
index 8250263..74cfde8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java
@@ -79,9 +79,9 @@ public class BPlusMetaIO extends PageIO {
     /**
      * @param buf Buffer.
      * @param lvl Level.
-     * @return Page reference at that level.
+     * @return First page ID at that level.
      */
-    public long getLeftmostPageId(ByteBuffer buf, int lvl) {
+    public long getFirstPageId(ByteBuffer buf, int lvl) {
         return buf.getLong(offset(lvl));
     }
 
@@ -90,12 +90,12 @@ public class BPlusMetaIO extends PageIO {
      * @param lvl    Level.
      * @param pageId Page ID.
      */
-    public void setLeftmostPageId(ByteBuffer buf, int lvl, long pageId) {
+    public void setFirstPageId(ByteBuffer buf, int lvl, long pageId) {
         assert lvl >= 0 && lvl < getLevelsCount(buf);
 
         buf.putLong(offset(lvl), pageId);
 
-        assert getLeftmostPageId(buf, lvl) == pageId;
+        assert getFirstPageId(buf, lvl) == pageId;
     }
 
     /**


[12/50] ignite git commit: ignite-db - tests wip

Posted by sb...@apache.org.
ignite-db - tests wip


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fdd09395
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fdd09395
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fdd09395

Branch: refs/heads/ignite-db-x-10884
Commit: fdd0939593bf65f32d8c37fce78f158adf83a78c
Parents: ff88620
Author: S.Vladykin <sv...@gridgain.com>
Authored: Fri Apr 22 17:41:43 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Fri Apr 22 17:41:43 2016 +0300

----------------------------------------------------------------------
 .../processors/database/BPlusTreeSelfTest.java      | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/fdd09395/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 4968fe8..e3919f1 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -263,6 +263,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testRandomRemove0() throws IgniteCheckedException {
+        // seed: 1461177795261173000, 1461187841179332000
         doTestRandomRemove(false);
     }
 
@@ -270,6 +271,7 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     public void testRandomRemove1() throws IgniteCheckedException {
+        // seed: 1461188744119034000 1461188844311788000 1461189099834526000
         doTestRandomRemove(true);
     }
 
@@ -284,6 +286,8 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         int cnt = 100_000;
 
+        int rmv = 0;
+
         for (long x = 0; x < cnt; x++)
             assertEquals(map.put(x,x), tree.put(x));
 
@@ -292,9 +296,17 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
                 Long x = (long)tree.randomInt(cnt);
 
                 if (map.remove(x) != null) {
+//                    if (rmv > 93440) {
+//                        X.println("Rmv: " + rmv + " -> " + x);
+//
+//                        if (rmv == 93449)
+//                            X.println(tree.printTree());
+//                    }
+
                     assertEquals(x, tree.remove(x));
                     assertNull(tree.remove(x));
 
+                    rmv++;
                     i++;
                 }
             }
@@ -320,6 +332,10 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
         }
     }
 
+    private void doTestStagedPutRemove(boolean canGetRow) {
+        // TODO
+    }
+
     /**
      * @param canGetRow Can get row from inner page.
      * @return Test tree instance.


[35/50] ignite git commit: ignite-db - minor

Posted by sb...@apache.org.
ignite-db - minor


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2b30543a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2b30543a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2b30543a

Branch: refs/heads/ignite-db-x-10884
Commit: 2b30543aa5cf777cfb5e3c87fcbb4882254c2c2d
Parents: 3daefae
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 01:08:35 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 01:08:35 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 51 +++++++++++---------
 1 file changed, 29 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/2b30543a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 06649cf..70f1af5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -965,20 +965,6 @@ public abstract class BPlusTree<L, T extends L> {
     }
 
     /**
-     * @param leftCnt Left count.
-     * @param rightCnt Right count.
-     * @param cap Capacity.
-     * @param leaf Is leaf pages.
-     * @return {@code true} If can merge these pages.
-     */
-    private boolean canMerge(int leftCnt, int rightCnt, int cap, boolean leaf) {
-        cap -= leftCnt;
-        cap -= rightCnt;
-
-        return cap > 0 || (cap == 0 && (leaf || leftCnt == 0 || rightCnt == 0));
-    }
-
-    /**
      * @param cnt Count.
      * @param cap Capacity.
      * @return {@code true} If may merge.
@@ -1628,7 +1614,7 @@ public abstract class BPlusTree<L, T extends L> {
         Tail<L> tail;
 
         /** */
-        List<FullPageId> emptyPages; // TODO May be use Object for single empty page
+        Object emptyPages;
 
         /** */
         byte needReplaceInner = FALSE;
@@ -1976,6 +1962,7 @@ public abstract class BPlusTree<L, T extends L> {
          * @param release Release write lock and release page.
          * @throws IgniteCheckedException If failed.
          */
+        @SuppressWarnings("unchecked")
         private void freePage(Page page, ByteBuffer buf, BPlusIO io, int lvl, boolean release)
             throws IgniteCheckedException {
             if (getFirstPageId(meta, lvl) == page.id()) {
@@ -1988,26 +1975,46 @@ public abstract class BPlusTree<L, T extends L> {
             // Mark removed.
             io.setRemoveId(buf, Long.MAX_VALUE);
 
+            if (release)
+                writeUnlockAndClose(page);
+
             if (reuseList == null)
                 return; // We are not allowed to reuse pages.
 
-            // We will reuse empty page.
+            // Reuse empty page.
             if (emptyPages == null)
-                emptyPages = new ArrayList<>(4);
+                emptyPages = page.fullId();
+            else {
+                List<FullPageId> list;
 
-            emptyPages.add(page.fullId());
+                if (emptyPages.getClass() != ArrayList.class) {
+                    list = new ArrayList<>(5);
 
-            if (release)
-                writeUnlockAndClose(page);
+                    list.add((FullPageId)emptyPages);
+
+                    emptyPages = list;
+                }
+                else
+                    list = (List<FullPageId>)emptyPages;
+
+                list.add(page.fullId());
+            }
         }
 
         /**
          * @throws IgniteCheckedException If failed.
          */
+        @SuppressWarnings("unchecked")
         private void reuseEmptyPages() throws IgniteCheckedException {
             if (emptyPages != null) {
-                for (int i = 0; i < emptyPages.size(); i++)
-                    reuseList.put(BPlusTree.this, emptyPages.get(i));
+                if (emptyPages.getClass() != ArrayList.class)
+                    reuseList.put(BPlusTree.this, (FullPageId)emptyPages);
+                else {
+                    List<FullPageId> list = (List<FullPageId>)emptyPages;
+
+                    for (int i = 0; i < list.size(); i++)
+                        reuseList.put(BPlusTree.this, list.get(i));
+                }
             }
         }
 


[09/50] ignite git commit: ignite-db - free

Posted by sb...@apache.org.
ignite-db - free


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e7d7e2cf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e7d7e2cf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e7d7e2cf

Branch: refs/heads/ignite-db-x-10884
Commit: e7d7e2cfad67c28ec448803239fd13ae8fbfa26e
Parents: d914e21
Author: S.Vladykin <sv...@gridgain.com>
Authored: Fri Apr 22 01:37:11 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Fri Apr 22 01:37:11 2016 +0300

----------------------------------------------------------------------
 .../processors/cache/database/RowStore.java     |  28 ++--
 .../cache/database/freelist/FreeItem.java       |  48 +------
 .../cache/database/freelist/FreeList.java       | 131 +++++++++++--------
 .../cache/database/freelist/FreeTree.java       |  24 +++-
 .../cache/database/freelist/io/FreeIO.java      |   8 +-
 .../cache/database/freelist/io/FreeInnerIO.java |  51 +++++---
 .../cache/database/freelist/io/FreeLeafIO.java  |  28 ++--
 .../cache/database/tree/io/DataPageIO.java      |  57 +++++---
 .../cache/database/tree/util/PageHandler.java   |   4 +-
 9 files changed, 210 insertions(+), 169 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
index 2ea2ee4..fdfadd6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
@@ -54,25 +54,30 @@ public class RowStore<T extends CacheDataRow> {
     private volatile long lastDataPageId;
 
     /** */
+    @Deprecated
     private final PageHandler<CacheDataRow> writeRow = new PageHandler<CacheDataRow>() {
         @Override public int run(Page page, ByteBuffer buf, CacheDataRow row, int ignore) throws IgniteCheckedException {
             int entrySize = DataPageIO.getEntrySize(coctx, row.key(), row.value());
 
             DataPageIO io = DataPageIO.VERSIONS.forPage(buf);
 
+            if (io.isEnoughSpace(buf, entrySize))
+                return -1;
+
             int idx = io.addRow(coctx, buf, row.key(), row.value(), row.version(), entrySize);
 
-            if (idx != -1) {
-                row.link(linkFromDwordOffset(page.id(), idx));
+            assert idx >= 0: idx;
 
-                assert row.link() != 0;
-            }
+            row.link(linkFromDwordOffset(page.id(), idx));
+
+            assert row.link() != 0;
 
             return idx;
         }
     };
 
     /** */
+    @Deprecated
     private final PageHandler<Void> rmvRow = new PageHandler<Void>() {
         @Override public int run(Page page, ByteBuffer buf, Void ignore, int itemId) throws IgniteCheckedException {
             DataPageIO io = DataPageIO.VERSIONS.forPage(buf);
@@ -125,9 +130,13 @@ public class RowStore<T extends CacheDataRow> {
     public void removeRow(long link) throws IgniteCheckedException {
         assert link != 0;
 
-        try (Page page = page(pageId(link))) {
-            writePage(page, rmvRow, null, dwordsOffset(link), 0);
+        if (freeList == null) {
+            try (Page page = page(pageId(link))) {
+                writePage(page, rmvRow, null, dwordsOffset(link), 0);
+            }
         }
+        else
+            freeList.removeRow(link);
     }
 
     /**
@@ -156,16 +165,17 @@ public class RowStore<T extends CacheDataRow> {
      */
     public void addRow(CacheDataRow row) throws IgniteCheckedException {
         if (freeList == null)
-            writeRowData0(row);
+            writeRowDataOld(row);
         else
-            freeList.writeRowData(row);
+            freeList.insertRow(row);
     }
 
     /**
      * @param row Row.
      * @throws IgniteCheckedException If failed.
      */
-    private void writeRowData0(CacheDataRow row) throws IgniteCheckedException {
+    @Deprecated
+    private void writeRowDataOld(CacheDataRow row) throws IgniteCheckedException {
         assert row.link() == 0;
 
         while (row.link() == 0) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeItem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeItem.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeItem.java
index 7de30c7..2c81074 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeItem.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeItem.java
@@ -26,38 +26,17 @@ public class FreeItem extends FullPageId {
     /** */
     private short freeSpace;
 
-    /** */
-    private short dispersion;
-
     /**
      * @param freeSpace Free space.
-     * @param dispersion Dispersion.
      * @param pageId  Page ID.
      * @param cacheId Cache ID.
      */
-    public FreeItem(short freeSpace, short dispersion, long pageId, int cacheId) {
+    public FreeItem(int freeSpace, long pageId, int cacheId) {
         super(pageId, cacheId);
 
-        assert freeSpace >= 0: freeSpace;
-
-        this.freeSpace = freeSpace;
-        this.dispersion = dispersion;
-    }
-
-    /**
-     * @param freeSpace Free space.
-     * @param dispersion Dispersion.
-     * @return Dispersed free space.
-     */
-    public static int disperse(int freeSpace, int dispersion) {
-        return (freeSpace << 16) | dispersion;
-    }
+        assert freeSpace >= 0 && freeSpace <= Short.MAX_VALUE: freeSpace;
 
-    /**
-     * @return Dispersed free space.
-     */
-    public int dispersedFreeSpace() {
-        return disperse(freeSpace, dispersion);
+        this.freeSpace = (short)freeSpace;
     }
 
     /**
@@ -66,25 +45,4 @@ public class FreeItem extends FullPageId {
     public short freeSpace() {
         return freeSpace;
     }
-
-    /**
-     * @param freeSpace Free space.
-     */
-    public void freeSpace(short freeSpace) {
-        this.freeSpace = freeSpace;
-    }
-
-    /**
-     * @return Dispersion.
-     */
-    public short dispersion() {
-        return dispersion;
-    }
-
-    /**
-     * @param dispersion Dispersion.
-     */
-    public void dispersion(short dispersion) {
-        this.dispersion = dispersion;
-    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
index a7507a0..68780c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
@@ -18,11 +18,11 @@
 package org.apache.ignite.internal.processors.cache.database.freelist;
 
 import java.nio.ByteBuffer;
-import java.util.concurrent.ThreadLocalRandom;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.Page;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
@@ -61,7 +61,41 @@ public class FreeList {
 
             assert idx >= 0;
 
-            return io.getFreeSpace(buf);
+            int freeSpace = io.getFreeSpace(buf);
+
+            // Put our free item.
+            tree(row.partition()).put(new FreeItem(freeSpace, page.id(), cctx.cacheId()));
+
+            return 0;
+        }
+    };
+
+    /** */
+    private final PageHandler<FreeTree> removeRow = new PageHandler<FreeTree>() {
+        @Override public int run(Page page, ByteBuffer buf, FreeTree tree, int itemId) throws IgniteCheckedException {
+            DataPageIO io = DataPageIO.VERSIONS.forPage(buf);
+
+            assert DataPageIO.check(itemId): itemId;
+
+            int oldFreeSpace = io.getFreeSpace(buf);
+
+            io.removeRow(buf, (byte)itemId);
+
+            int newFreeSpace = io.getFreeSpace(buf);
+
+            // Move page to the new position with respect to the new free space.
+            FreeItem item = tree.remove(new FreeItem(oldFreeSpace, page.id(), cctx.cacheId()));
+
+            // If item is null, then it was removed concurrently by insertRow, because
+            // in removeRow we own the write lock on this page. Thus we can be sure that
+            // insertRow will update position correctly after us.
+            if (item != null) {
+                FreeItem old = tree.put(new FreeItem(newFreeSpace, page.id(), cctx.cacheId()));
+
+                assert old == null;
+            }
+
+            return 0;
         }
     };
 
@@ -83,14 +117,12 @@ public class FreeList {
 
     /**
      * @param tree Tree.
-     * @param neededSpace Needed free space.
+     * @param lookupItem Lookup item.
      * @return Free item or {@code null} if it was impossible to find one.
      * @throws IgniteCheckedException If failed.
      */
-    private FreeItem take(FreeTree tree, short neededSpace) throws IgniteCheckedException {
-        assert neededSpace > 0 && neededSpace < Short.MAX_VALUE: neededSpace;
-
-        FreeItem res = tree.removeCeil(new FreeItem(neededSpace, dispersion(), 0, 0));
+    private FreeItem take(FreeTree tree, FreeItem lookupItem) throws IgniteCheckedException {
+        FreeItem res = tree.removeCeil(lookupItem);
 
         assert res == null || (res.pageId() != 0 && res.cacheId() == cctx.cacheId()): res;
 
@@ -98,35 +130,28 @@ public class FreeList {
     }
 
     /**
-     * @return Random dispersion value.
-     */
-    private static short dispersion() {
-        return (short)ThreadLocalRandom.current().nextInt(Short.MIN_VALUE, Short.MAX_VALUE);
-    }
-
-    /**
-     * @param part Partition.
+     * @param partId Partition.
      * @return Tree.
      * @throws IgniteCheckedException If failed.
      */
-    private FreeTree tree(Integer part) throws IgniteCheckedException {
-        assert part >= 0 && part < Short.MAX_VALUE: part;
+    private FreeTree tree(Integer partId) throws IgniteCheckedException {
+        assert partId >= 0 && partId < Short.MAX_VALUE: partId;
 
-        GridFutureAdapter<FreeTree> fut = trees.get(part);
+        GridFutureAdapter<FreeTree> fut = trees.get(partId);
 
         if (fut == null) {
             fut = new GridFutureAdapter<>();
 
-            if (trees.putIfAbsent(part, fut) != null)
-                fut = trees.get(part);
+            if (trees.putIfAbsent(partId, fut) != null)
+                fut = trees.get(partId);
             else {
                 // Index name will be the same across restarts.
-                String idxName = part + "$$" + cctx.cacheId() + "_free";
+                String idxName = partId + "$$" + cctx.cacheId() + "_free";
 
                 IgniteBiTuple<FullPageId,Boolean> t = cctx.shared().database().meta()
                     .getOrAllocateForIndex(cctx.cacheId(), idxName);
 
-                fut.onDone(new FreeTree(reuseList, cctx.cacheId(), pageMem, t.get1(), t.get2()));
+                fut.onDone(new FreeTree(reuseList, cctx.cacheId(), partId, pageMem, t.get1(), t.get2()));
             }
         }
 
@@ -134,59 +159,51 @@ public class FreeList {
     }
 
     /**
+     * @param link Row link.
+     * @throws IgniteCheckedException
+     */
+    public void removeRow(long link) throws IgniteCheckedException {
+        assert link != 0;
+
+        long pageId = PageIdUtils.pageId(link);
+        int itemId = PageIdUtils.dwordsOffset(link);
+
+        try (Page page = pageMem.page(new FullPageId(pageId, cctx.cacheId()))) {
+            writePage(page, removeRow, null, itemId, -1);
+        }
+    }
+
+    /**
      * @param row Row.
      * @throws IgniteCheckedException If failed.
      */
-    public void writeRowData(CacheDataRow row) throws IgniteCheckedException {
-        // assert row.link == 0;
+    public void insertRow(CacheDataRow row) throws IgniteCheckedException {
+        assert row.link() == 0: row.link();
 
         int entrySize = DataPageIO.getEntrySize(cctx.cacheObjectContext(), row.key(), row.value());
 
         assert entrySize > 0 && entrySize < Short.MAX_VALUE: entrySize;
 
         FreeTree tree = tree(row.partition());
-        FreeItem item = take(tree, (short)entrySize);
 
-        Page page = null;
-        int freeSpace = -1;
+        // TODO add random pageIndex here for lower contention?
+        FreeItem item = take(tree, new FreeItem(entrySize, 0, cctx.cacheId()));
 
-        try {
+        try (Page page = item == null ?
+            allocateDataPage(row.partition()) :
+            pageMem.page(item)
+        ) {
             if (item == null) {
                 DataPageIO io = DataPageIO.VERSIONS.latest();
 
-                page = allocatePage(row.partition());
-
                 ByteBuffer buf = page.getForInitialWrite();
 
                 io.initNewPage(buf, page.id());
 
-                freeSpace = writeRow.run(page, buf, row, entrySize);
-            }
-            else {
-                page = pageMem.page(item);
-
-                freeSpace = writePage(page, writeRow, row, entrySize, -1);
-            }
-        }
-        finally {
-            if (page != null) {
-                page.close();
-
-                if (freeSpace != -1) { // Put back to the tree.
-                    assert freeSpace >= 0 && freeSpace < Short.MAX_VALUE: freeSpace;
-
-                    if (item == null)
-                        item = new FreeItem((short)freeSpace, dispersion(), page.id(), cctx.cacheId());
-                    else {
-                        item.freeSpace((short)freeSpace);
-                        item.dispersion(dispersion());
-                    }
-
-                    FreeItem old = tree.put(item);
-
-                    assert old == null;
-                }
+                writeRow.run(page, buf, row, entrySize);
             }
+            else
+                writePage(page, writeRow, row, entrySize, -1);
         }
     }
 
@@ -195,7 +212,7 @@ public class FreeList {
      * @return Page.
      * @throws IgniteCheckedException If failed.
      */
-    private Page allocatePage(int part) throws IgniteCheckedException {
+    private Page allocateDataPage(int part) throws IgniteCheckedException {
         FullPageId pageId = pageMem.allocatePage(cctx.cacheId(), part, PageIdAllocator.FLAG_DATA);
 
         return pageMem.page(pageId);

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
index 8af7561..c5138c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeTree.java
@@ -35,6 +35,9 @@ import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList
  * Data structure for data pages and their free spaces.
  */
 public class FreeTree extends BPlusTree<FreeItem, FreeItem> {
+    /** */
+    private final int partId;
+
     /**
      * @param reuseList Reuse list.
      * @param cacheId Cache ID.
@@ -43,16 +46,25 @@ public class FreeTree extends BPlusTree<FreeItem, FreeItem> {
      * @param initNew    Initialize new index.
      * @throws IgniteCheckedException If failed.
      */
-    public FreeTree(ReuseList reuseList, int cacheId, PageMemory pageMem, FullPageId metaPageId, boolean initNew)
+    public FreeTree(ReuseList reuseList, int cacheId, int partId, PageMemory pageMem, FullPageId metaPageId, boolean initNew)
         throws IgniteCheckedException {
         super(cacheId, pageMem, metaPageId, reuseList);
 
+        this.partId = partId;
+
         assert pageMem != null;
 
         if (initNew)
             initNew();
     }
 
+    /**
+     * @return Partition ID.
+     */
+    public int getPartId() {
+        return partId;
+    }
+
     /** {@inheritDoc} */
     @Override protected BPlusIO<FreeItem> io(int type, int ver) {
         if (type == PageIO.T_FREE_INNER)
@@ -76,12 +88,12 @@ public class FreeTree extends BPlusTree<FreeItem, FreeItem> {
     /** {@inheritDoc} */
     @Override protected int compare(BPlusIO<FreeItem> io, ByteBuffer buf, int idx, FreeItem row)
         throws IgniteCheckedException {
-        if (io.isLeaf()) // In a leaf we can do a fair compare.
-            return Short.compare(((FreeIO)io).freeSpace(buf, idx), row.freeSpace());
+        int res = Short.compare(((FreeIO)io).getFreeSpace(buf, idx), row.freeSpace());
+
+        if (res == 0)
+            res = Long.compare(((FreeLeafIO)io).getPageIndex(buf, idx), FreeInnerIO.pageIndex(row.pageId()));
 
-        // In inner pages we do compare on dispersed free space to avoid contention on a single page
-        // when all the entries are equal and many pages have the same free space.
-        return Integer.compare(((FreeIO)io).dispersedFreeSpace(buf, idx), row.dispersedFreeSpace());
+        return res;
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeIO.java
index de28b22..d893b27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeIO.java
@@ -26,14 +26,14 @@ public interface FreeIO {
     /**
      * @param buf Buffer.
      * @param idx Index.
-     * @return Dispersed free space.
+     * @return Free space.
      */
-    public int dispersedFreeSpace(ByteBuffer buf, int idx);
+    public short getFreeSpace(ByteBuffer buf, int idx);
 
     /**
      * @param buf Buffer.
      * @param idx Index.
-     * @return Free space.
+     * @return Page index.
      */
-    public short freeSpace(ByteBuffer buf, int idx);
+    public int getPageIndex(ByteBuffer buf, int idx);
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeInnerIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeInnerIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeInnerIO.java
index b9d13b8..14c7a00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeInnerIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeInnerIO.java
@@ -1,7 +1,10 @@
 package org.apache.ignite.internal.processors.cache.database.freelist.io;
 
 import java.nio.ByteBuffer;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.processors.cache.database.freelist.FreeItem;
+import org.apache.ignite.internal.processors.cache.database.freelist.FreeTree;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
@@ -20,48 +23,62 @@ public class FreeInnerIO extends BPlusInnerIO<FreeItem> implements FreeIO {
      * @param ver Page format version.
      */
     protected FreeInnerIO(int ver) {
-        super(T_FREE_INNER, ver, false, 4); // freeSpace(2) + dispersion(2)
+        super(T_FREE_INNER, ver, false, 6); // freeSpace(2) + pageIndex(4)
+    }
+
+    /**
+     * @param pageId Page Id.
+     * @return Page index.
+     */
+    public static int pageIndex(long pageId) {
+        long idx = PageIdUtils.pageIdx(pageId);
+
+        assert idx >= 0 && idx < Integer.MAX_VALUE: idx;
+
+        return (int)idx;
     }
 
     /** {@inheritDoc} */
     @Override public void store(ByteBuffer buf, int idx, FreeItem row) {
-        store(buf, idx, row.dispersedFreeSpace());
+        int off = offset(idx);
+
+        buf.putShort(off, row.freeSpace());
+        buf.putInt(off + 2, pageIndex(row.pageId()));
     }
 
     /** {@inheritDoc} */
     @Override public void store(ByteBuffer dst, int dstIdx, BPlusIO<FreeItem> srcIo, ByteBuffer src, int srcIdx) {
-        store(dst, dstIdx, ((FreeIO)srcIo).dispersedFreeSpace(src, srcIdx));
-    }
+        FreeIO srcFreeIo = (FreeIO)srcIo;
 
-    /**
-     * @param buf Buffer.
-     * @param idx Index.
-     * @param dispersedFreeSpace Dispersed free space.
-     */
-    private void store(ByteBuffer buf, int idx, int dispersedFreeSpace) {
-        int off = offset(idx);
+        int off = offset(dstIdx);
 
-        buf.putInt(off, dispersedFreeSpace);
+        dst.putShort(off, srcFreeIo.getFreeSpace(src, srcIdx));
+        dst.putInt(off + 2, srcFreeIo.getPageIndex(src, srcIdx));
     }
 
     /** {@inheritDoc} */
-    @Override public int dispersedFreeSpace(ByteBuffer buf, int idx) {
+    @Override public short getFreeSpace(ByteBuffer buf, int idx) {
         int off = offset(idx);
 
-        return buf.getInt(off);
+        return buf.getShort(off);
     }
 
     /** {@inheritDoc} */
-    @Override public short freeSpace(ByteBuffer buf, int idx) {
+    @Override public int getPageIndex(ByteBuffer buf, int idx) {
         int off = offset(idx);
 
-        return buf.getShort(off);
+        return buf.getInt(off + 2);
     }
 
     /** {@inheritDoc} */
     @Override public FreeItem getLookupRow(BPlusTree<FreeItem, ?> tree, ByteBuffer buf, int idx) {
         int off = offset(idx);
 
-        return new FreeItem(buf.getShort(off), buf.getShort(off + 2), 0, 0);
+        short freeSpace = buf.getShort(off);
+        int pageIdx = buf.getInt(off + 2);
+
+        long pageId = PageIdUtils.pageId(((FreeTree)tree).getPartId(), PageIdAllocator.FLAG_DATA, pageIdx);
+
+        return new FreeItem(freeSpace, pageId, tree.getCacheId());
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
index 5496897..8503c7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/FreeLeafIO.java
@@ -18,10 +18,13 @@
 package org.apache.ignite.internal.processors.cache.database.freelist.io;
 
 import java.nio.ByteBuffer;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.database.freelist.FreeItem;
+import org.apache.ignite.internal.processors.cache.database.freelist.FreeTree;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeItem;
 
 /**
  * Routines for free list leaf pages.
@@ -36,35 +39,40 @@ public class FreeLeafIO extends BPlusLeafIO<FreeItem> implements FreeIO {
      * @param ver Page format version.
      */
     protected FreeLeafIO(int ver) {
-        super(T_FREE_LEAF, ver, 12); // freeSpace(2) + dispersion(2) + pageId(8)
+        super(T_FREE_LEAF, ver, 6); // freeSpace(2) + pageIndex(4)
     }
 
     /** {@inheritDoc} */
-    @Override public final void store(ByteBuffer buf, int idx, FreeItem row) {
+    @Override public void store(ByteBuffer buf, int idx, FreeItem row) {
         int off = offset(idx);
 
-        buf.putInt(off, row.dispersedFreeSpace());
-        buf.putLong(off + 4, row.pageId());
+        buf.putShort(off, row.freeSpace());
+        buf.putInt(off + 2, FreeInnerIO.pageIndex(row.pageId()));
     }
 
     /** {@inheritDoc} */
-    @Override public int dispersedFreeSpace(ByteBuffer buf, int idx) {
+    @Override public short getFreeSpace(ByteBuffer buf, int idx) {
         int off = offset(idx);
 
-        return buf.getInt(off);
+        return buf.getShort(off);
     }
 
     /** {@inheritDoc} */
-    @Override public short freeSpace(ByteBuffer buf, int idx) {
+    @Override public int getPageIndex(ByteBuffer buf, int idx) {
         int off = offset(idx);
 
-        return buf.getShort(off);
+        return buf.getInt(off + 2);
     }
 
     /** {@inheritDoc} */
     @Override public FreeItem getLookupRow(BPlusTree<FreeItem, ?> tree, ByteBuffer buf, int idx) {
         int off = offset(idx);
 
-        return new FreeItem(buf.getShort(off), buf.getShort(off + 2), buf.getLong(off + 4), tree.getCacheId());
+        short freeSpace = buf.getShort(off);
+        int pageIdx = buf.getInt(off + 2);
+
+        long pageId = PageIdUtils.pageId(((FreeTree)tree).getPartId(), PageIdAllocator.FLAG_DATA, pageIdx);
+
+        return new FreeItem(freeSpace, pageId, tree.getCacheId());
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java
index beaed49..a5efb5d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java
@@ -250,9 +250,9 @@ public class DataPageIO extends PageIO {
             assert indirectItemIdx >= directCnt && indirectItemIdx < directCnt + indirectCnt: indirectCnt;
 
             itemId = directItemIndex(getItem(buf, indirectItemIdx));
-        }
 
-        assert itemId >= 0 && itemId < directCnt: itemId; // Direct item must be here.
+            assert itemId >= 0 && itemId < directCnt: itemId; // Direct item must be here.
+        }
 
         return toOffset(getItem(buf, itemId));
     }
@@ -393,10 +393,17 @@ public class DataPageIO extends PageIO {
             return; // TODO May be have a separate list of free pages?
         }
 
+        // Get the entry size before the actual remove.
+        int rmvEntrySize = getEntrySize(buf, getDataOffset(buf, itemId), false);
+
         if (itemId < directCnt)
             removeDirectItem(buf, itemId, directCnt, indirectCnt);
         else
             removeIndirectItem(buf, itemId, directCnt, indirectCnt);
+
+        // Increase free space.
+        setFreeSpace(buf, getFreeSpace(buf) + rmvEntrySize +
+            ITEM_SIZE * (directCnt - getDirectCount(buf) + indirectCnt - getIndirectCount(buf)));
     }
 
     /**
@@ -482,14 +489,24 @@ public class DataPageIO extends PageIO {
     }
 
     /**
-     * @param entrySize Entry size as returned by {@link #getEntrySize(int, int)}.
-     * @param firstOff First entry data offset.
+     * @param newEntrySizeWithItem New entry size as returned by {@link #getEntrySize(int, int)}.
+     * @param firstEntryOff First entry data offset.
      * @param directCnt Direct items count.
      * @param indirectCnt Indirect items count.
      * @return {@code true} If there is enough space for the entry.
      */
-    private static boolean enoughSpaceForEntry(int entrySize, int firstOff, int directCnt, int indirectCnt) {
-        return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstOff - entrySize;
+    public static boolean isEnoughSpace(int newEntrySizeWithItem, int firstEntryOff, int directCnt, int indirectCnt) {
+        return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstEntryOff - newEntrySizeWithItem;
+    }
+
+    /**
+     * @param buf Buffer.
+     * @param newEntrySizeWithItem New entry size as returned by {@link #getEntrySize(int, int)}.
+     * @return {@code true} If there is enough space for the entry.
+     */
+    public boolean isEnoughSpace(ByteBuffer buf, int newEntrySizeWithItem) {
+        return isEnoughSpace(newEntrySizeWithItem,
+            getFirstEntryOffset(buf), getDirectCount(buf), getIndirectCount(buf));
     }
 
     /**
@@ -498,7 +515,7 @@ public class DataPageIO extends PageIO {
      * @param key Key.
      * @param val Value.
      * @param ver Version.
-     * @param entrySize Entry size as returned by {@link #getEntrySize(int, int)}.
+     * @param entrySizeWithItem Entry size as returned by {@link #getEntrySize(int, int)}.
      * @return Item ID.
      * @throws IgniteCheckedException If failed.
      */
@@ -508,29 +525,26 @@ public class DataPageIO extends PageIO {
         CacheObject key,
         CacheObject val,
         GridCacheVersion ver,
-        int entrySize
+        int entrySizeWithItem
     ) throws IgniteCheckedException {
-        if (entrySize > buf.capacity() - ITEMS_OFF) // TODO span multiple data pages with a single large entry
+        if (entrySizeWithItem > buf.capacity() - ITEMS_OFF) // TODO span multiple data pages with a single large entry
             throw new IgniteException("Too big entry: " + key + " " + val);
 
         int directCnt = getDirectCount(buf);
         int indirectCnt = getIndirectCount(buf);
-
         int dataOff = getFirstEntryOffset(buf);
 
-        // Compact if we do not have enough space.
-        if (!enoughSpaceForEntry(entrySize, dataOff, directCnt, indirectCnt)) {
+        // Compact if we do not have enough space for entry.
+        if (!isEnoughSpace(entrySizeWithItem, dataOff, directCnt, indirectCnt)) {
             dataOff = compactDataEntries(buf, directCnt);
 
-//            assert enoughSpaceForEntry(entrySize, dataOff, directCnt, indirectCnt);
-            if (!enoughSpaceForEntry(entrySize, dataOff, directCnt, indirectCnt))
-                return -1; // TODO replace with assert
+            assert isEnoughSpace(entrySizeWithItem, dataOff, directCnt, indirectCnt);
         }
 
-        // Attempt to write data right before the first entry.
-        dataOff -= entrySize - ITEM_SIZE;
+        // Write data right before the first entry.
+        dataOff -= entrySizeWithItem - ITEM_SIZE;
 
-        writeRowData(coctx, buf, dataOff, entrySize, key, val, ver);
+        writeRowData(coctx, buf, dataOff, entrySizeWithItem, key, val, ver);
 
         setFirstEntryOffset(buf, dataOff);
 
@@ -538,6 +552,11 @@ public class DataPageIO extends PageIO {
 
         assert check(itemId): itemId;
 
+        // Update free space. If number of direct items did not change, then we were able to reuse item slot.
+        setFreeSpace(buf, getFreeSpace(buf) - entrySizeWithItem + (getDirectCount(buf) == directCnt ? ITEM_SIZE : 0));
+
+        assert getFreeSpace(buf) >= 0;
+
         return (byte)itemId;
     }
 
@@ -653,7 +672,7 @@ public class DataPageIO extends PageIO {
      * @param val Value.
      * @param ver Version.
      */
-    public void writeRowData(
+    private void writeRowData(
         CacheObjectContext coctx,
         ByteBuffer buf,
         int dataOff,

http://git-wip-us.apache.org/repos/asf/ignite/blob/e7d7e2cf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java
index 3d1b4e4..4e1ff0d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java
@@ -85,7 +85,7 @@ public abstract class PageHandler<X> {
     public static <X> int writePage(Page page, PageHandler<X> h, X arg, int intArg, int dfltRes)
         throws IgniteCheckedException {
         if (page == null)
-            return dfltRes;
+            return dfltRes; // TODO drop dfltRes parameter and assert not null
 
         int res;
 
@@ -94,7 +94,7 @@ public abstract class PageHandler<X> {
         ByteBuffer buf = page.getForWrite();
 
         if (buf == null)
-            return dfltRes;
+            return dfltRes; // TODO ---//---
 
         try {
             res = h.run(page, buf, arg, intArg);


[45/50] ignite git commit: Merge remote-tracking branch 'community/ignite-db-2' into ignite-db-2

Posted by sb...@apache.org.
Merge remote-tracking branch 'community/ignite-db-2' into ignite-db-2


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d67fa205
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d67fa205
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d67fa205

Branch: refs/heads/ignite-db-x-10884
Commit: d67fa205da391bcccc46e5122b43a00ce934f10c
Parents: 66fb8ef 50e452c
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Apr 27 00:33:10 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Apr 27 00:33:10 2016 -0700

----------------------------------------------------------------------
 .../ignite/internal/processors/query/h2/database/io/H2InnerIO.java | 2 --
 .../ignite/internal/processors/query/h2/database/io/H2LeafIO.java  | 2 --
 2 files changed, 4 deletions(-)
----------------------------------------------------------------------



[31/50] ignite git commit: GG-11044 - Page evictions.

Posted by sb...@apache.org.
GG-11044 - Page evictions.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2da9ee37
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2da9ee37
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2da9ee37

Branch: refs/heads/ignite-db-x-10884
Commit: 2da9ee377e92e70b647e84e0d021c86db620b7c8
Parents: d7dfb91
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Apr 26 10:34:50 2016 -0700
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Apr 26 10:34:50 2016 -0700

----------------------------------------------------------------------
 .../internal/pagemem/impl/PageMemoryImpl.java   | 26 --------------------
 1 file changed, 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/2da9ee37/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
index 7863fbc..4422962 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryImpl.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.ArrayList;
-import java.util.BitSet;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -1059,31 +1058,6 @@ public class  PageMemoryImpl implements PageMemory {
     }
 
     /**
-     * Long set based on BitSet.
-     */
-    private static class SimpleLongSet {
-        /** */
-        final BitSet bitSet = new BitSet();
-
-        /**
-         * Add value to set.
-         *
-         * @param val Value.
-         */
-        void add(final long val) {
-            bitSet.set((int)(val % Integer.MAX_VALUE));
-        }
-
-        /**
-         * @param val Value to check.
-         * @return {@code True} if set contains value.
-         */
-        boolean contains(final long val) {
-            return bitSet.get((int)(val % Integer.MAX_VALUE));
-        }
-    }
-
-    /**
      *
      */
     private class Segment extends ReentrantReadWriteLock {


[03/50] ignite git commit: ignite-db - reuse fix

Posted by sb...@apache.org.
ignite-db - reuse fix


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/813c36f6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/813c36f6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/813c36f6

Branch: refs/heads/ignite-db-x-10884
Commit: 813c36f6959ab2216dc6c32c20ef81777289a53e
Parents: 9cfda8d
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 20 23:14:21 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 20 23:14:21 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 25 ++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/813c36f6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 51649a5..98fd8eb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -853,6 +853,8 @@ public abstract class BPlusTree<L, T extends L> {
         finally {
             r.releaseTail();
             r.releaseMeta();
+
+            r.reuseEmptyPages();
         }
     }
 
@@ -1608,6 +1610,9 @@ public abstract class BPlusTree<L, T extends L> {
         Tail<L> tail;
 
         /** */
+        List<FullPageId> emptyPages;
+
+        /** */
         byte needReplaceInner = FALSE;
 
         /** */
@@ -1893,11 +1898,23 @@ public abstract class BPlusTree<L, T extends L> {
             // Mark removed.
             io.setRemoveId(buf, Long.MAX_VALUE);
 
-            // Reuse empty page.
-            if (reuseList != null) {
-//            U.dumpStack("page: " + page.fullId());
+            if (reuseList == null)
+                return; // We are not allowed to reuse pages.
 
-                reuseList.put(BPlusTree.this, page.fullId());
+            // We will reuse empty page.
+            if (emptyPages == null)
+                emptyPages = new ArrayList<>(4);
+
+            emptyPages.add(page.fullId());
+        }
+
+        /**
+         * @throws IgniteCheckedException If failed.
+         */
+        private void reuseEmptyPages() throws IgniteCheckedException {
+            if (emptyPages != null) {
+                for (int i = 0; i < emptyPages.size(); i++)
+                    reuseList.put(BPlusTree.this, emptyPages.get(i));
             }
         }
 


[27/50] ignite git commit: ignite-db - more fixes

Posted by sb...@apache.org.
ignite-db - more fixes


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a11f9f0e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a11f9f0e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a11f9f0e

Branch: refs/heads/ignite-db-x-10884
Commit: a11f9f0e9b4799dcea6bf1edb3a3618d56e3ded7
Parents: 892f1f5
Author: S.Vladykin <sv...@gridgain.com>
Authored: Tue Apr 26 07:27:00 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Tue Apr 26 07:27:00 2016 +0300

----------------------------------------------------------------------
 .../processors/cache/database/tree/BPlusTree.java    | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a11f9f0e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index f0919d2..eb91602 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -1706,7 +1706,7 @@ public abstract class BPlusTree<L, T extends L> {
             while (t.lvl != 0) { // If we've found empty branch, merge it top down.
                 boolean res = merge(t);
 
-                assert res;
+                assert res: needMergeEmptyBranch;
 
                 if (needMergeEmptyBranch == TRUE)
                     needMergeEmptyBranch = READY; // Need to mark that we've already done the first iteration.
@@ -1913,7 +1913,7 @@ public abstract class BPlusTree<L, T extends L> {
             int leftCnt = left.getCount();
             int rightCnt = right.getCount();
 
-            assert prntCnt > 0: prntCnt;
+            assert prntCnt > 0 || needMergeEmptyBranch == READY: prntCnt;
 
             int newCnt = leftCnt + rightCnt;
 
@@ -2048,7 +2048,10 @@ public abstract class BPlusTree<L, T extends L> {
          * @throws IgniteCheckedException If failed.
          */
         private boolean merge(Tail<L> prnt) throws IgniteCheckedException {
-            if (prnt.getCount() == 0)
+            // If we are merging empty branch this is acceptable because even if we merge
+            // two routing pages, one of them is effectively dropped in this merge, so just
+            // keep a single routing page.
+            if (prnt.getCount() == 0 && needMergeEmptyBranch != READY)
                 return false; // Parent is an empty routing page, child forward page will have another parent.
 
             Tail<L> right = prnt.down;
@@ -2097,12 +2100,6 @@ public abstract class BPlusTree<L, T extends L> {
          * Release pages for all locked levels at the tail.
          */
         private void releaseTail() {
-//            U.dumpStack("releaseTail");
-//            X.println("------>");
-//            for (Tail<L> t = tail; t != null; t = t.down)
-//                X.println("" + t);
-//            X.println("------<");
-
             doReleaseTail(tail);
 
             tail = null;


[37/50] ignite git commit: ignite-db - reuse tests

Posted by sb...@apache.org.
ignite-db - reuse tests


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bbcf8fe4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bbcf8fe4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bbcf8fe4

Branch: refs/heads/ignite-db-x-10884
Commit: bbcf8fe4024dfec42cb33e3ff61bc6ba94d072df
Parents: c523c6f
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 02:58:12 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 02:58:12 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 10 ++++
 .../cache/database/tree/reuse/ReuseList.java    | 17 +++++--
 .../cache/database/tree/reuse/ReuseTree.java    |  2 +-
 .../database/tree/reuse/io/ReuseInnerIO.java    |  2 +-
 .../database/tree/reuse/io/ReuseLeafIO.java     |  2 +-
 .../database/BPlusTreeReuseSelfTest.java        | 34 +++++++++++++
 .../processors/database/BPlusTreeSelfTest.java  | 52 +++++++++++++++-----
 7 files changed, 101 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/bbcf8fe4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index e712a41..df853c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -1001,6 +1001,16 @@ public abstract class BPlusTree<L, T extends L> {
      }
 
     /**
+     * @return Root level.
+     * @throws IgniteCheckedException If failed.
+     */
+    public final int rootLevel() throws IgniteCheckedException {
+        try (Page meta = page(metaPageId)) {
+            return getRootLevel(meta);
+        }
+    }
+
+    /**
      * TODO may produce wrong results on concurrent access
      *
      * @return Size.

http://git-wip-us.apache.org/repos/asf/ignite/blob/bbcf8fe4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
index a40fa18..f494dad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
@@ -28,7 +28,7 @@ import org.apache.ignite.lang.IgniteBiTuple;
 /**
  * Reuse list for index pages.
  */
-public class ReuseList {
+public final class ReuseList {
     /** */
     private static final FullPageId MIN = new FullPageId(0,0);
 
@@ -60,12 +60,23 @@ public class ReuseList {
     }
 
     /**
+     * @return Size.
+     * @throws IgniteCheckedException If failed.
+     */
+    public long size() throws IgniteCheckedException {
+        long size = 0;
+
+        for (ReuseTree tree : trees)
+            size += tree.size();
+
+        return size;
+    }
+
+    /**
      * @param client Client.
      * @return Reuse tree.
      */
     private ReuseTree tree(BPlusTree<?,?> client) {
-        assert trees.length > 1;
-
         int treeIdx = client.randomInt(trees.length);
 
         ReuseTree tree = trees[treeIdx];

http://git-wip-us.apache.org/repos/asf/ignite/blob/bbcf8fe4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseTree.java
index 6206465..e7e6a1b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseTree.java
@@ -33,7 +33,7 @@ import org.apache.ignite.internal.processors.cache.database.tree.reuse.io.ReuseL
 /**
  * Reuse tree for index pages.
  */
-public class ReuseTree extends BPlusTree<FullPageId, FullPageId> {
+public final class ReuseTree extends BPlusTree<FullPageId, FullPageId> {
     /**
      * @param reuseList Reuse list.
      * @param cacheId Cache ID.

http://git-wip-us.apache.org/repos/asf/ignite/blob/bbcf8fe4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseInnerIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseInnerIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseInnerIO.java
index 18313e7..522efba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseInnerIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseInnerIO.java
@@ -29,7 +29,7 @@ import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
 /**
  * Reuse list inner page IO routines.
  */
-public class ReuseInnerIO extends BPlusInnerIO<FullPageId> {
+public final class ReuseInnerIO extends BPlusInnerIO<FullPageId> {
     /** */
     public static final IOVersions<ReuseInnerIO> VERSIONS = new IOVersions<>(
         new ReuseInnerIO(1)

http://git-wip-us.apache.org/repos/asf/ignite/blob/bbcf8fe4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
index bf46024..3d2d37a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/io/ReuseLeafIO.java
@@ -28,7 +28,7 @@ import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
 /**
  * Reuse list leaf page IO routines.
  */
-public class ReuseLeafIO extends BPlusLeafIO<FullPageId> {
+public final class ReuseLeafIO extends BPlusLeafIO<FullPageId> {
     /** */
     public static final IOVersions<ReuseLeafIO> VERSIONS = new IOVersions<>(
         new ReuseLeafIO(1)

http://git-wip-us.apache.org/repos/asf/ignite/blob/bbcf8fe4/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java
new file mode 100644
index 0000000..a70bb1d
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.database.MetaStore;
+import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+
+/**
+ * Test with reuse.
+ */
+public class BPlusTreeReuseSelfTest extends BPlusTreeSelfTest {
+    /** {@inheritDoc} */
+    @Override protected ReuseList createReuseList(int cacheId, PageMemory pageMem, int segments, MetaStore metaStore)
+        throws IgniteCheckedException {
+        return new ReuseList(cacheId, pageMem, segments, metaStore);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/bbcf8fe4/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 813fdc3..7258f36 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -27,13 +27,16 @@ import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.impl.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.database.MetaStore;
 import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
 import org.apache.ignite.internal.util.lang.GridCursor;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -75,10 +78,10 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /** */
     private ReuseList reuseList;
 
-    @Override
-    protected long getTestTimeout() {
-        return 25 * 60 * 1000;
-    }
+//    /** {@inheritDoc} */
+//    @Override protected long getTestTimeout() {
+//        return 25 * 60 * 1000;
+//    }
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
@@ -92,17 +95,35 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
 
         pageMem.start();
 
-        reuseList = null;
-//            new ReuseList(CACHE_ID, pageMem, 2, new MetaStore() {
-//            @Override public IgniteBiTuple<FullPageId,Boolean> getOrAllocateForIndex(int cacheId, String idxName)
-//                throws IgniteCheckedException {
-//                return new T2<>(allocatePage(), true);
-//            }
-//        });
+        reuseList = createReuseList(CACHE_ID, pageMem, 2, new MetaStore() {
+            @Override public IgniteBiTuple<FullPageId,Boolean> getOrAllocateForIndex(int cacheId, String idxName)
+                throws IgniteCheckedException {
+                return new T2<>(allocatePage(), true);
+            }
+        });
+    }
+
+    /**
+     * @param cacheId Cache ID.
+     * @param pageMem Page memory.
+     * @param segments Segments.
+     * @param metaStore Store.
+     * @return Reuse list.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected ReuseList createReuseList(int cacheId, PageMemory pageMem, int segments, MetaStore metaStore)
+        throws IgniteCheckedException {
+        return null;
     }
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
+        if (reuseList != null) {
+            long size = reuseList.size();
+
+            assertTrue("Reuse size: " + size, size < CNT);
+        }
+
         pageMem.stop();
 
         MAX_PER_PAGE = 0;
@@ -438,6 +459,8 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
         }
 
         assertFalse(tree.find(null, null).next());
+        assertEquals(0, tree.size());
+        assertEquals(0, tree.rootLevel());
     }
 
     /**
@@ -505,7 +528,12 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
      * @throws IgniteCheckedException If failed.
      */
     private TestTree createTestTree(boolean canGetRow) throws IgniteCheckedException {
-        return new TestTree(reuseList, canGetRow, CACHE_ID, pageMem, allocatePage());
+        TestTree tree = new TestTree(reuseList, canGetRow, CACHE_ID, pageMem, allocatePage());
+
+        assertEquals(0, tree.size());
+        assertEquals(0, tree.rootLevel());
+
+        return tree;
     }
 
     /**


[29/50] ignite git commit: ignite-db - 20

Posted by sb...@apache.org.
ignite-db - 20


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b0cab0c5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b0cab0c5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b0cab0c5

Branch: refs/heads/ignite-db-x-10884
Commit: b0cab0c54edb1347b12a953ef92802ab43fd6714
Parents: 92e5315
Author: S.Vladykin <sv...@gridgain.com>
Authored: Tue Apr 26 07:32:43 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Tue Apr 26 07:32:43 2016 +0300

----------------------------------------------------------------------
 .../processors/database/BPlusTreeSelfTest.java  | 32 ++++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/b0cab0c5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 54accf5..ce3840a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -114,9 +114,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_mm_1() throws IgniteCheckedException {
+    public void testPutRemove_1_20_mm_1() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = -1;
         RMV_INC = -1;
 
@@ -126,9 +126,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_mm_0() throws IgniteCheckedException {
+    public void testPutRemove_1_20_mm_0() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = -1;
         RMV_INC = -1;
 
@@ -138,9 +138,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_pm_1() throws IgniteCheckedException {
+    public void testPutRemove_1_20_pm_1() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = 1;
         RMV_INC = -1;
 
@@ -150,9 +150,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_pm_0() throws IgniteCheckedException {
+    public void testPutRemove_1_20_pm_0() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = 1;
         RMV_INC = -1;
 
@@ -162,9 +162,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_pp_1() throws IgniteCheckedException {
+    public void testPutRemove_1_20_pp_1() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = 1;
         RMV_INC = 1;
 
@@ -174,9 +174,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_pp_0() throws IgniteCheckedException {
+    public void testPutRemove_1_20_pp_0() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = 1;
         RMV_INC = 1;
 
@@ -186,9 +186,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_mp_1() throws IgniteCheckedException {
+    public void testPutRemove_1_20_mp_1() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = -1;
         RMV_INC = 1;
 
@@ -198,9 +198,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
     /**
      * @throws IgniteCheckedException If failed.
      */
-    public void testPutRemove_1_10_mp_0() throws IgniteCheckedException {
+    public void testPutRemove_1_20_mp_0() throws IgniteCheckedException {
         MAX_ITEMS_COUNT = 1;
-        CNT = 10;
+        CNT = 20;
         PUT_INC = -1;
         RMV_INC = 1;
 


[07/50] ignite git commit: ignite-db - fixes

Posted by sb...@apache.org.
ignite-db - fixes


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d914e213
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d914e213
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d914e213

Branch: refs/heads/ignite-db-x-10884
Commit: d914e213a36029d9ee650f6f6936f55f470a6bfe
Parents: d7e35ee
Author: S.Vladykin <sv...@gridgain.com>
Authored: Thu Apr 21 01:18:19 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Thu Apr 21 01:18:19 2016 +0300

----------------------------------------------------------------------
 .../cache/database/tree/BPlusTree.java          | 22 +++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/d914e213/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 61c49d7..1e72677 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -432,7 +432,7 @@ public abstract class BPlusTree<L, T extends L> {
 
             Tail<L> t = r.getTail(lvl, false);
 
-            assert t.io == io : "must be the same"; // Otherwise may be not compatible.
+            assert t.io == io : "must be the same"; // Otherwise can be not compatible.
 
             return r.mergePages(prnt, t, fwd, fwdBuf) ? TRUE : FALSE;
         }
@@ -1204,9 +1204,13 @@ public abstract class BPlusTree<L, T extends L> {
             try {
                 BPlusIO<L> io = io(buf);
 
-                assert io.getCount(buf) > 0;
+                assert io.getCount(buf) >= 0; // Count can be 0 if it is a routing page.
 
-                return inner(io).getLeft(buf, 0);
+                long res = inner(io).getLeft(buf, 0);
+
+                assert res != 0: "inner page with no route down: " + page.fullId();
+
+                return res;
             }
             finally {
                 page.releaseRead();
@@ -1936,14 +1940,14 @@ public abstract class BPlusTree<L, T extends L> {
             // Remove found inner key from inner page.
             doRemove(inner.io, inner.page, inner.buf, cnt, inner.idx, inner.lvl, kickLeft);
 
-            // If inner page was root and became empty, it was freed in doRemove.
+            // If inner page was root and became empty, it was handled in doRemove.
             // Otherwise we can be sure that inner page was not freed, at lead it must become
             // an empty routing page. Thus always starting from inner.down here.
             for (Tail t = inner.down; t != null; t = t.down) {
                 if (t.fwd != null)
                     t = t.fwd;
 
-                assert t.io.getCount(t.buf) == 0;
+                assert t.io.getCount(t.buf) == 0: row;
 
                 freePage(t.page, t.buf, t.io, t.lvl);
             }
@@ -2051,21 +2055,23 @@ public abstract class BPlusTree<L, T extends L> {
                 }
             }
             else {
-                assert cur.io == back.io: "must always be the same"; // Otherwise may be not compatible.
+                assert cur.io == back.io: "must always be the same"; // Otherwise can be not compatible.
 
                 if (mergePages(prnt, back, cur.page, cur.buf)) {
                     assert prnt.down == back;
                     assert back.fwd == cur;
 
-                    // Back becomes current.
+                    // Back becomes current, current is dropped.
                     back.down = cur.down;
                     back.fwd = null;
 
+                    // Always unlock and release current because we made it invisible for further code.
+                    writeUnlockAndClose(cur.page);
+
                     if (releaseMerged) {
                         prnt.down = null;
 
                         writeUnlockAndClose(back.page);
-                        writeUnlockAndClose(cur.page);
                     }
 
                     return true;


[38/50] ignite git commit: ignite-db - reuse bag

Posted by sb...@apache.org.
ignite-db - reuse bag


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/83d7a5ca
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/83d7a5ca
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/83d7a5ca

Branch: refs/heads/ignite-db-x-10884
Commit: 83d7a5caee33e0e77a8ef5ef0eb9ada80b7da760
Parents: bbcf8fe
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 27 05:46:56 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 27 05:46:56 2016 +0300

----------------------------------------------------------------------
 .../cache/database/freelist/FreeList.java       |   2 +-
 .../cache/database/tree/BPlusTree.java          | 163 ++++++++++++-------
 .../cache/database/tree/reuse/ReuseBag.java     |  35 ++++
 .../cache/database/tree/reuse/ReuseList.java    |  33 +++-
 4 files changed, 170 insertions(+), 63 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/83d7a5ca/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
index 68780c2..30ed519 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
@@ -122,7 +122,7 @@ public class FreeList {
      * @throws IgniteCheckedException If failed.
      */
     private FreeItem take(FreeTree tree, FreeItem lookupItem) throws IgniteCheckedException {
-        FreeItem res = tree.removeCeil(lookupItem);
+        FreeItem res = tree.removeCeil(lookupItem, null);
 
         assert res == null || (res.pageId() != 0 && res.cacheId() == cctx.cacheId()): res;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/83d7a5ca/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index df853c0..c0144a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache.database.tree;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -36,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusMetaIO;
 import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseBag;
 import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
 import org.apache.ignite.internal.util.lang.GridCursor;
@@ -278,7 +280,7 @@ public abstract class BPlusTree<L, T extends L> {
                 return Put.RETRY;
 
             // Do insert.
-            L moveUpRow = insert(p.meta, io, buf, p.row, idx, p.rightId, lvl);
+            L moveUpRow = insert(p.bag, p.meta, io, buf, p.row, idx, p.rightId, lvl);
 
             // Check if split happened.
             if (moveUpRow != null) {
@@ -526,7 +528,7 @@ public abstract class BPlusTree<L, T extends L> {
 
             io.initNewPage(buf, metaPageId);
 
-            try (Page root = allocatePage()) {
+            try (Page root = allocatePage(null)) {
                 latestLeafIO().initNewPage(root.getForInitialWrite(), root.id());
 
                 io.setLevelsCount(buf, 1);
@@ -811,13 +813,14 @@ public abstract class BPlusTree<L, T extends L> {
 
     /**
      * @param row Lookup row.
+     * @param bag Reuse bag.
      * @return Removed row.
      * @throws IgniteCheckedException If failed.
      */
-    public final T removeCeil(L row) throws IgniteCheckedException {
+    public final T removeCeil(L row, ReuseBag bag) throws IgniteCheckedException {
         assert row != null;
 
-        return doRemove(row, true);
+        return doRemove(row, true, bag);
     }
 
     /**
@@ -828,17 +831,18 @@ public abstract class BPlusTree<L, T extends L> {
     public final T remove(L row) throws IgniteCheckedException {
         assert row != null;
 
-        return doRemove(row, false);
+        return doRemove(row, false, null);
     }
 
     /**
      * @param row Lookup row.
      * @param ceil If we can remove ceil row when we can not find exact.
+     * @param bag Reuse bag.
      * @return Removed row.
      * @throws IgniteCheckedException If failed.
      */
-    private T doRemove(L row, boolean ceil) throws IgniteCheckedException {
-        Remove r = new Remove(row, ceil);
+    private T doRemove(L row, boolean ceil, ReuseBag bag) throws IgniteCheckedException {
+        Remove r = new Remove(row, ceil, bag);
 
         try {
             for (;;) {
@@ -865,7 +869,7 @@ public abstract class BPlusTree<L, T extends L> {
             r.releaseTail();
             r.releaseMeta();
 
-            r.reuseEmptyPages();
+            r.reuseFreePages();
         }
     }
 
@@ -1057,7 +1061,17 @@ public abstract class BPlusTree<L, T extends L> {
      * @throws IgniteCheckedException If failed.
      */
     public final T put(T row) throws IgniteCheckedException {
-        Put p = new Put(row);
+        return put(row, null);
+    }
+
+    /**
+     * @param row Row.
+     * @param bag Reuse bag.
+     * @return Old row.
+     * @throws IgniteCheckedException If failed.
+     */
+    public final T put(T row, ReuseBag bag) throws IgniteCheckedException {
+        Put p = new Put(row, bag);
 
         try {
             for (;;) { // Go down with retries.
@@ -1141,6 +1155,7 @@ public abstract class BPlusTree<L, T extends L> {
     }
 
     /**
+     * @param bag Reuse bag.
      * @param meta Meta page.
      * @param io IO.
      * @param buf Buffer.
@@ -1151,9 +1166,9 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Move up row.
      * @throws IgniteCheckedException If failed.
      */
-    private L insertWithSplit(Page meta, BPlusIO<L> io, final ByteBuffer buf, L row,
+    private L insertWithSplit(ReuseBag bag, Page meta, BPlusIO<L> io, final ByteBuffer buf, L row,
         int idx, long rightId, int lvl) throws IgniteCheckedException {
-        try (Page fwd = allocatePage()) {
+        try (Page fwd = allocatePage(bag)) {
             // Need to check this before the actual split, because after the split we will have new forward page here.
             boolean hadFwd = io.getForward(buf) != 0;
 
@@ -1186,7 +1201,7 @@ public abstract class BPlusTree<L, T extends L> {
             if (!hadFwd && lvl == getRootLevel(meta)) { // We are splitting root.
                 long newRootId;
 
-                try (Page newRoot = allocatePage()) {
+                try (Page newRoot = allocatePage(bag)) {
                     newRootId = newRoot.id();
 
                     if (io.isLeaf())
@@ -1215,6 +1230,7 @@ public abstract class BPlusTree<L, T extends L> {
     }
 
     /**
+     * @param bag Reuse bag.
      * @param meta Meta page.
      * @param io IO.
      * @param buf Buffer.
@@ -1225,13 +1241,13 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Move up row.
      * @throws IgniteCheckedException If failed.
      */
-    private L insert(Page meta, BPlusIO<L> io, ByteBuffer buf, L row, int idx, long rightId, int lvl)
+    private L insert(ReuseBag bag, Page meta, BPlusIO<L> io, ByteBuffer buf, L row, int idx, long rightId, int lvl)
         throws IgniteCheckedException {
         int maxCnt = io.getMaxCount(buf);
         int cnt = io.getCount(buf);
 
         if (cnt == maxCnt) // Need to split page.
-            return insertWithSplit(meta, io, buf, row, idx, rightId, lvl);
+            return insertWithSplit(bag, meta, io, buf, row, idx, rightId, lvl);
 
         insertSimple(io, buf, row, idx, rightId);
 
@@ -1595,11 +1611,17 @@ public abstract class BPlusTree<L, T extends L> {
         /** */
         byte needReplaceInner = FALSE;
 
+        /** */
+        ReuseBag bag;
+
         /**
          * @param row Row.
+         * @param bag Reuse bag.
          */
-        private Put(T row) {
+        private Put(T row, ReuseBag bag) {
             super(row);
+
+            this.bag = bag;
         }
 
         /** {@inheritDoc} */
@@ -1657,7 +1679,7 @@ public abstract class BPlusTree<L, T extends L> {
     /**
      * Remove operation.
      */
-    private final class Remove extends Get {
+    private final class Remove extends Get implements ReuseBag {
         /** */
         boolean ceil;
 
@@ -1665,9 +1687,6 @@ public abstract class BPlusTree<L, T extends L> {
         Tail<L> tail;
 
         /** */
-        Object emptyPages;
-
-        /** */
         byte needReplaceInner = FALSE;
 
         /** */
@@ -1682,14 +1701,72 @@ public abstract class BPlusTree<L, T extends L> {
         /** */
         short innerIdx = Short.MIN_VALUE;
 
+        /** */
+        Object freePages;
+
+        /** */
+        ReuseBag bag;
+
         /**
          * @param row Row.
          * @param ceil If we can remove ceil row when we can not find exact.
          */
-        private Remove(L row, boolean ceil) {
+        private Remove(L row, boolean ceil, ReuseBag bag) {
             super(row);
 
             this.ceil = ceil;
+            this.bag = bag;
+        }
+
+        /**
+         * @return Reuse bag.
+         */
+        private ReuseBag bag() {
+            return bag != null ? bag : this;
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("unchecked")
+        @Override public FullPageId pollFreePage() {
+            assert bag == null;
+
+            if (freePages == null)
+                return null;
+
+            if (freePages.getClass() == ArrayDeque.class)
+                return ((ArrayDeque<FullPageId>)freePages).poll();
+
+            FullPageId res = (FullPageId)freePages;
+
+            freePages = null;
+
+            return res;
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("unchecked")
+        @Override public void addFreePage(FullPageId pageId) {
+            assert pageId != null;
+            assert bag == null;
+
+            if (freePages == null)
+                freePages = pageId;
+            else {
+                ArrayDeque<Object> queue;
+
+                if (freePages.getClass() == ArrayDeque.class)
+                    queue = (ArrayDeque<Object>)freePages;
+                else {
+                    assert freePages instanceof FullPageId;
+
+                    queue = new ArrayDeque<>();
+
+                    queue.add(freePages);
+                    freePages = queue;
+                }
+
+                queue.add(pageId);
+            }
         }
 
         /** {@inheritDoc} */
@@ -2029,44 +2106,17 @@ public abstract class BPlusTree<L, T extends L> {
             if (release)
                 writeUnlockAndClose(page);
 
-            if (reuseList == null)
-                return; // We are not allowed to reuse pages.
-
-            // Reuse empty page.
-            if (emptyPages == null)
-                emptyPages = page.fullId();
-            else {
-                List<FullPageId> list;
-
-                if (emptyPages.getClass() != ArrayList.class) {
-                    list = new ArrayList<>(5);
-
-                    list.add((FullPageId)emptyPages);
-
-                    emptyPages = list;
-                }
-                else
-                    list = (List<FullPageId>)emptyPages;
-
-                list.add(page.fullId());
-            }
+            bag().addFreePage(page.fullId());
         }
 
         /**
          * @throws IgniteCheckedException If failed.
          */
         @SuppressWarnings("unchecked")
-        private void reuseEmptyPages() throws IgniteCheckedException {
-            if (emptyPages != null) {
-                if (emptyPages.getClass() != ArrayList.class)
-                    reuseList.put(BPlusTree.this, (FullPageId)emptyPages);
-                else {
-                    List<FullPageId> list = (List<FullPageId>)emptyPages;
-
-                    for (int i = 0; i < list.size(); i++)
-                        reuseList.put(BPlusTree.this, list.get(i));
-                }
-            }
+        private void reuseFreePages() throws IgniteCheckedException {
+            // If we have a bag, then it will be processed at the upper level.
+            if (reuseList != null && bag == null)
+                reuseList.add(BPlusTree.this, this);
         }
 
         /**
@@ -2403,13 +2453,14 @@ public abstract class BPlusTree<L, T extends L> {
     }
 
     /**
+     * @param bag Reuse bag.
      * @return Allocated page.
      */
-    private Page allocatePage() throws IgniteCheckedException {
-        FullPageId pageId = null;
+    private Page allocatePage(ReuseBag bag) throws IgniteCheckedException {
+        FullPageId pageId = bag != null ? bag.pollFreePage() : null;
 
-        if (reuseList != null)
-            pageId = reuseList.take(this);
+        if (pageId == null && reuseList != null)
+            pageId = reuseList.take(this, bag);
 
         if (pageId == null)
             pageId = pageMem.allocatePage(cacheId, -1, PageIdAllocator.FLAG_IDX);

http://git-wip-us.apache.org/repos/asf/ignite/blob/83d7a5ca/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java
new file mode 100644
index 0000000..7fce7b7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.database.tree.reuse;
+
+import org.apache.ignite.internal.pagemem.FullPageId;
+
+/**
+ * Reuse bag for free index pages.
+ */
+public interface ReuseBag {
+    /**
+     * @param pageId Free page ID for reuse.
+     */
+    public void addFreePage(FullPageId pageId);
+
+    /**
+     * @return Free page ID for reuse or {@code null} if empty.
+     */
+    public FullPageId pollFreePage();
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/83d7a5ca/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
index f494dad..eb89da9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
@@ -98,21 +98,42 @@ public final class ReuseList {
 
     /**
      * @param client Client tree.
+     * @param bag Reuse bag.
      * @return Page ID.
      * @throws IgniteCheckedException If failed.
      */
-    public FullPageId take(BPlusTree<?,?> client) throws IgniteCheckedException {
+    public FullPageId take(BPlusTree<?,?> client, ReuseBag bag) throws IgniteCheckedException {
+        if (trees == null)
+            return null;
+
+        ReuseTree tree = tree(client);
+
         // Remove and return page at min possible position.
-        return trees == null ? null : tree(client).removeCeil(MIN);
+        FullPageId res = tree.removeCeil(MIN, bag);
+
+        assert res != null || tree.size() == 0; // TODO remove
+
+        return res;
     }
 
     /**
-     * @param fullPageId Page ID.
+     * @param client Client tree.
+     * @param bag Reuse bag.
      * @throws IgniteCheckedException If failed.
      */
-    public void put(BPlusTree<?,?> client, FullPageId fullPageId) throws IgniteCheckedException {
-        assert fullPageId != null;
+    public void add(BPlusTree<?,?> client, ReuseBag bag) throws IgniteCheckedException {
+        if (bag == null)
+            return;
 
-        tree(client).put(fullPageId);
+        ReuseTree tree = tree(client);
+
+        for (;;) {
+            FullPageId pageId = bag.pollFreePage();
+
+            if (pageId == null)
+                break;
+
+            tree.put(pageId, bag);
+        }
     }
 }


[02/50] ignite git commit: ignite-db - final

Posted by sb...@apache.org.
ignite-db - final


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9cfda8d6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9cfda8d6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9cfda8d6

Branch: refs/heads/ignite-db-x-10884
Commit: 9cfda8d61aa34e0c43b86f1e0a5cb2f8961be438
Parents: 14ffbf0
Author: S.Vladykin <sv...@gridgain.com>
Authored: Wed Apr 20 21:20:14 2016 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Wed Apr 20 21:20:14 2016 +0300

----------------------------------------------------------------------
 .../processors/cache/database/tree/BPlusTree.java       | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/9cfda8d6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
index 37d6d06..51649a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
@@ -596,7 +596,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Cursor.
      * @throws IgniteCheckedException If failed.
      */
-    public GridCursor<T> find(L lower, L upper) throws IgniteCheckedException {
+    public final GridCursor<T> find(L lower, L upper) throws IgniteCheckedException {
         if (lower == null)
             return findNoLower(upper);
 
@@ -612,7 +612,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Found row.
      */
     @SuppressWarnings("unchecked")
-    public T findOne(L row) throws IgniteCheckedException {
+    public final T findOne(L row) throws IgniteCheckedException {
         GetOne g = new GetOne(row);
 
         doFind(g);
@@ -803,7 +803,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Removed row.
      * @throws IgniteCheckedException If failed.
      */
-    public T removeCeil(L row) throws IgniteCheckedException {
+    public final T removeCeil(L row) throws IgniteCheckedException {
         assert row != null;
 
         return remove(row, true);
@@ -814,7 +814,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Removed row.
      * @throws IgniteCheckedException If failed.
      */
-    public T remove(L row) throws IgniteCheckedException {
+    public final T remove(L row) throws IgniteCheckedException {
         assert row != null;
 
         return remove(row, false);
@@ -826,7 +826,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Removed row.
      * @throws IgniteCheckedException If failed.
      */
-    public T remove(L row, boolean ceil) throws IgniteCheckedException {
+    public final T remove(L row, boolean ceil) throws IgniteCheckedException {
         Remove r = new Remove(row, ceil);
 
         try {
@@ -1005,7 +1005,7 @@ public abstract class BPlusTree<L, T extends L> {
      * @return Old row.
      * @throws IgniteCheckedException If failed.
      */
-    public T put(T row) throws IgniteCheckedException {
+    public final T put(T row) throws IgniteCheckedException {
         Put p = new Put(row);
 
         try {