You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by bu...@apache.org on 2015/08/07 01:33:42 UTC
incubator-asterixdb-hyracks git commit: improve the buffer cache
perf. with 1) a better hash function for fileid-pageid,
2) reduce synchronization in clock page replacement policy.
Repository: incubator-asterixdb-hyracks
Updated Branches:
refs/heads/master b5e88a2a3 -> fd434262d
improve the buffer cache perf. with 1) a better hash function for fileid-pageid, 2) reduce synchronization in clock page replacement policy.
Change-Id: I296c589a556a9afa7f27c6f560fa07fc4e2c1861
Reviewed-on: https://asterix-gerrit.ics.uci.edu/342
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Reviewed-by: Young-Seok Kim <ki...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/commit/fd434262
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/tree/fd434262
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/diff/fd434262
Branch: refs/heads/master
Commit: fd434262d10525f0fb9c0088659f9410fd67a8bf
Parents: b5e88a2
Author: Yingyi Bu <bu...@gmail.com>
Authored: Thu Aug 6 11:46:03 2015 -0700
Committer: Yingyi Bu <bu...@gmail.com>
Committed: Thu Aug 6 16:30:07 2015 -0700
----------------------------------------------------------------------
.../am/lsm/common/impls/VirtualBufferCache.java | 7 +--
.../storage/common/buffercache/BufferCache.java | 2 +-
.../ClockPageReplacementStrategy.java | 53 +++++++++-----------
3 files changed, 30 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/fd434262/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index d5d04fb..62bf025 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -50,9 +50,9 @@ public class VirtualBufferCache implements IVirtualBufferCache {
this.allocator = allocator;
this.fileMapManager = new TransientFileMapManager();
this.pageSize = pageSize;
- this.numPages = numPages;
+ this.numPages = 2 * (numPages / 2) + 1;
- buckets = new CacheBucket[numPages];
+ buckets = new CacheBucket[this.numPages];
pages = new ArrayList<VirtualPage>();
nextFree = 0;
open = false;
@@ -186,7 +186,8 @@ public class VirtualBufferCache implements IVirtualBufferCache {
}
private int hash(long dpid) {
- return (int) (dpid % buckets.length);
+ int hashValue = (int) dpid ^ (Integer.reverse((int) (dpid >>> 32)) >>> 1);
+ return hashValue % buckets.length;
}
private VirtualPage getOrAllocPage(long dpid) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/fd434262/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
index 0ceab64..d973947 100644
--- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
@@ -441,7 +441,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
}
private int hash(long dpid) {
- int hashValue = (int) (dpid ^ (dpid >>> 32));
+ int hashValue = (int) dpid ^ (Integer.reverse((int) (dpid >>> 32)) >>> 1);
return hashValue % pageMap.length;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/fd434262/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
index 611bf48..b5edc8d 100644
--- a/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
+++ b/hyracks/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
@@ -3,9 +3,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* you may obtain a copy of the License from
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,22 +15,19 @@
package edu.uci.ics.hyracks.storage.common.buffercache;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.atomic.AtomicInteger;
public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
private static final int MAX_UNSUCCESSFUL_CYCLE_COUNT = 3;
- private final Lock lock;
private IBufferCacheInternal bufferCache;
private int clockPtr;
private ICacheMemoryAllocator allocator;
- private int numPages = 0;
+ private AtomicInteger numPages = new AtomicInteger(0);
private final int pageSize;
private final int maxAllowedNumPages;
public ClockPageReplacementStrategy(ICacheMemoryAllocator allocator, int pageSize, int maxAllowedNumPages) {
- this.lock = new ReentrantLock();
this.allocator = allocator;
this.pageSize = pageSize;
this.maxAllowedNumPages = maxAllowedNumPages;
@@ -59,16 +56,13 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
@Override
public ICachedPageInternal findVictim() {
- lock.lock();
ICachedPageInternal cachedPage = null;
- try {
- if (numPages >= maxAllowedNumPages) {
- cachedPage = findVictimByEviction();
- } else {
- cachedPage = allocatePage();
- }
- } finally {
- lock.unlock();
+ int pageCount = getNumPages();
+ // pageCount is a lower-bound of numPages.
+ if (pageCount >= maxAllowedNumPages) {
+ cachedPage = findVictimByEviction();
+ } else {
+ cachedPage = allocatePage();
}
return cachedPage;
}
@@ -93,7 +87,10 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
return cPage;
}
}
- clockPtr = (clockPtr + 1) % numPages;
+ /**
+ * The clockPtr may miss the last added pages in this round.
+ */
+ clockPtr = (clockPtr + 1) % getNumPages();
if (clockPtr == startClockPtr) {
++cycleCount;
}
@@ -101,22 +98,22 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
return null;
}
+ /**
+ * The number returned here could only be smaller or equal to the actual number
+ * of pages, because numPages is monotonically incremented.
+ */
@Override
public int getNumPages() {
- int retNumPages = 0;
- lock.lock();
- try {
- retNumPages = numPages;
- } finally {
- lock.unlock();
- }
- return retNumPages;
+ return numPages.get();
}
private ICachedPageInternal allocatePage() {
- CachedPage cPage = new CachedPage(numPages, allocator.allocate(pageSize, 1)[0], this);
- bufferCache.addPage(cPage);
- numPages++;
+ CachedPage cPage = null;
+ synchronized (this) {
+ cPage = new CachedPage(numPages.get(), allocator.allocate(pageSize, 1)[0], this);
+ bufferCache.addPage(cPage);
+ numPages.incrementAndGet();
+ }
AtomicBoolean accessedFlag = getPerPageObject(cPage);
if (!accessedFlag.compareAndSet(true, false)) {
if (cPage.pinIfGoodVictim()) {