You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org> on 2016/03/11 02:48:01 UTC
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/705
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
---
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
M asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
12 files changed, 92 insertions(+), 57 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/705/1
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
index 5532e79..905854d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
@@ -189,7 +189,8 @@
initializeResourceIdFactory();
datasetLifecycleManager = new DatasetLifecycleManager(storageProperties, localResourceRepository,
- MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager());
+ MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager(),
+ ioManager.getIODevices().size());
isShuttingdown = false;
@@ -361,8 +362,8 @@
}
@Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
- return datasetLifecycleManager.getVirtualBufferCaches(datasetID);
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int partition) {
+ return datasetLifecycleManager.getVirtualBufferCaches(datasetID, partition);
}
@Override
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
index b975970..83f03f9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
@@ -88,8 +88,8 @@
}
@Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
- return asterixAppRuntimeContext.getVirtualBufferCaches(datasetID);
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int partition) {
+ return asterixAppRuntimeContext.getVirtualBufferCaches(datasetID, partition);
}
@Override
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
index 496a10e..c16ed31 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -75,7 +75,7 @@
public double getBloomFilterFalsePositiveRate();
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int partition);
public Object getFeedManager();
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java b/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
index 3b4617a..a99aead 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
@@ -81,9 +81,10 @@
* creates (if necessary) and returns the dataset virtual buffer caches.
*
* @param datasetID
+ * @param partition
* @return
*/
- List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
+ List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int partition);
/**
* Flushes then closes all open datasets
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java b/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
index c58b013..2675686 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
@@ -37,7 +37,7 @@
@Override
public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx) {
return ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject())
- .getVirtualBufferCaches(datasetID);
+ .getVirtualBufferCaches(datasetID, ctx.getTaskAttemptId().getTaskId().getPartition());
}
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index 0cd88d6..270656a 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -53,7 +53,7 @@
public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeCycleComponent {
private final AsterixStorageProperties storageProperties;
- private final Map<Integer, List<IVirtualBufferCache>> datasetVirtualBufferCaches;
+ private final Map<Integer, DatasetVirtualBufferCaches> datasetVirtualBufferCachesMap;
private final Map<Integer, ILSMOperationTracker> datasetOpTrackers;
private final Map<Integer, DatasetInfo> datasetInfos;
private final ILocalResourceRepository resourceRepository;
@@ -62,14 +62,17 @@
private long used;
private final ILogManager logManager;
private final LogRecord logRecord;
+ private final int numPartitions;
public DatasetLifecycleManager(AsterixStorageProperties storageProperties,
- ILocalResourceRepository resourceRepository, int firstAvilableUserDatasetID, ILogManager logManager) {
+ ILocalResourceRepository resourceRepository, int firstAvilableUserDatasetID,
+ ILogManager logManager, int numPartitions) {
this.logManager = logManager;
this.storageProperties = storageProperties;
this.resourceRepository = resourceRepository;
this.firstAvilableUserDatasetID = firstAvilableUserDatasetID;
- datasetVirtualBufferCaches = new HashMap<Integer, List<IVirtualBufferCache>>();
+ this.numPartitions = numPartitions;
+ datasetVirtualBufferCachesMap = new HashMap<>();
datasetOpTrackers = new HashMap<Integer, ILSMOperationTracker>();
datasetInfos = new HashMap<Integer, DatasetInfo>();
capacity = storageProperties.getMemoryComponentGlobalBudget();
@@ -303,45 +306,36 @@
return openIndexesInfo;
}
- @Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
- synchronized (datasetVirtualBufferCaches) {
- List<IVirtualBufferCache> vbcs = datasetVirtualBufferCaches.get(datasetID);
+ private DatasetVirtualBufferCaches getVirtualBufferCaches(int datasetID) {
+ synchronized (datasetVirtualBufferCachesMap) {
+ DatasetVirtualBufferCaches vbcs = datasetVirtualBufferCachesMap.get(datasetID);
if (vbcs == null) {
- initializeDatasetVirtualBufferCache(datasetID);
- vbcs = datasetVirtualBufferCaches.get(datasetID);
- if (vbcs == null) {
- throw new IllegalStateException("Could not find dataset " + datasetID + " virtual buffer cache.");
- }
+ vbcs = initializeDatasetVirtualBufferCache(datasetID);
}
return vbcs;
}
}
+ @Override
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int partition) {
+ DatasetVirtualBufferCaches dvbcs = getVirtualBufferCaches(datasetID);
+ return dvbcs.getVirtualBufferCaches(partition);
+ }
+
private void removeDatasetFromCache(int datasetID) throws HyracksDataException {
deallocateDatasetMemory(datasetID);
datasetInfos.remove(datasetID);
- datasetVirtualBufferCaches.remove(datasetID);
+ datasetVirtualBufferCachesMap.remove(datasetID);
datasetOpTrackers.remove(datasetID);
}
- private void initializeDatasetVirtualBufferCache(int datasetID) {
- List<IVirtualBufferCache> vbcs = new ArrayList<IVirtualBufferCache>();
- synchronized (datasetVirtualBufferCaches) {
- int numPages = datasetID < firstAvilableUserDatasetID
- ? storageProperties.getMetadataMemoryComponentNumPages()
- : storageProperties.getMemoryComponentNumPages();
- for (int i = 0; i < storageProperties.getMemoryComponentsNum(); i++) {
- MultitenantVirtualBufferCache vbc = new MultitenantVirtualBufferCache(
- new VirtualBufferCache(new ResourceHeapBufferAllocator(this, Integer.toString(datasetID)),
- storageProperties.getMemoryComponentPageSize(),
- numPages / storageProperties.getMemoryComponentsNum()));
- vbcs.add(vbc);
- }
- datasetVirtualBufferCaches.put(datasetID, vbcs);
+ private DatasetVirtualBufferCaches initializeDatasetVirtualBufferCache(int datasetID) {
+ synchronized (datasetVirtualBufferCachesMap) {
+ DatasetVirtualBufferCaches dvbcs = new DatasetVirtualBufferCaches(datasetID);
+ datasetVirtualBufferCachesMap.put(datasetID, dvbcs);
+ return dvbcs;
}
}
-
@Override
public ILSMOperationTracker getOperationTracker(int datasetID) {
synchronized (datasetOpTrackers) {
@@ -354,7 +348,7 @@
}
}
- private abstract class Info {
+ private static abstract class Info {
protected int referenceCount;
protected boolean isOpen;
@@ -372,7 +366,7 @@
}
}
- public class IndexInfo extends Info {
+ public static class IndexInfo extends Info {
private final ILSMIndex index;
private final long resourceId;
private final int datasetId;
@@ -396,7 +390,7 @@
}
}
- public class DatasetInfo extends Info implements Comparable<DatasetInfo> {
+ public static class DatasetInfo extends Info implements Comparable<DatasetInfo> {
private final Map<Long, IndexInfo> indexes;
private final int datasetID;
private long lastAccess;
@@ -637,7 +631,7 @@
closeAllDatasets();
- datasetVirtualBufferCaches.clear();
+ datasetVirtualBufferCachesMap.clear();
datasetOpTrackers.clear();
datasetInfos.clear();
}
@@ -685,11 +679,7 @@
synchronized (dsInfo) {
// This is not needed for external datasets' indexes since they never use the virtual buffer cache.
if (!dsInfo.memoryAllocated && !dsInfo.isExternal) {
- List<IVirtualBufferCache> vbcs = getVirtualBufferCaches(dsInfo.datasetID);
- long additionalSize = 0;
- for (IVirtualBufferCache vbc : vbcs) {
- additionalSize += vbc.getNumPages() * vbc.getPageSize();
- }
+ long additionalSize = getVirtualBufferCaches(dsInfo.datasetID).getTotalSize();
while (used + additionalSize > capacity) {
if (!evictCandidateDataset()) {
throw new HyracksDataException("Cannot allocate dataset " + dsInfo.datasetID
@@ -710,10 +700,7 @@
}
synchronized (dsInfo) {
if (dsInfo.isOpen && dsInfo.memoryAllocated) {
- List<IVirtualBufferCache> vbcs = getVirtualBufferCaches(dsInfo.datasetID);
- for (IVirtualBufferCache vbc : vbcs) {
- used -= (vbc.getNumPages() * vbc.getPageSize());
- }
+ used -= getVirtualBufferCaches(dsInfo.datasetID).getTotalSize();
dsInfo.memoryAllocated = false;
}
}
@@ -725,4 +712,50 @@
int did = Integer.parseInt(resourcePath);
allocateDatasetMemory(did);
}
+
+ private class DatasetVirtualBufferCaches {
+ private final int datasetID;
+ private final Map<Integer, List<IVirtualBufferCache>> partitionVirtualBufferCaches = new HashMap<>();
+
+ public DatasetVirtualBufferCaches(int datasetID) {
+ this.datasetID = datasetID;
+ }
+
+ private List<IVirtualBufferCache> initializeVirtualBufferCaches(int partition) {
+ synchronized (partitionVirtualBufferCaches) {
+ int numPages = datasetID < firstAvilableUserDatasetID
+ ? storageProperties.getMetadataMemoryComponentNumPages()
+ : storageProperties.getMemoryComponentNumPages();
+ List<IVirtualBufferCache> vbcs = new ArrayList<>();
+ for (int i = 0; i < storageProperties.getMemoryComponentsNum(); i++) {
+ MultitenantVirtualBufferCache vbc = new MultitenantVirtualBufferCache(
+ new VirtualBufferCache(new ResourceHeapBufferAllocator(DatasetLifecycleManager.this,
+ Integer.toString(datasetID)), storageProperties.getMemoryComponentPageSize(),
+ numPages / storageProperties.getMemoryComponentsNum() / numPartitions));
+ vbcs.add(vbc);
+ }
+ partitionVirtualBufferCaches.put(partition, vbcs);
+ return vbcs;
+ }
+ }
+
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int partition) {
+ synchronized (partitionVirtualBufferCaches) {
+ List<IVirtualBufferCache> vbcs = partitionVirtualBufferCaches.get(partition);
+ if (vbcs == null) {
+ vbcs = initializeVirtualBufferCaches(partition);
+ }
+ return vbcs;
+ }
+ }
+
+ public long getTotalSize() {
+ int numPages = datasetID < firstAvilableUserDatasetID
+ ? storageProperties.getMetadataMemoryComponentNumPages()
+ : storageProperties.getMemoryComponentNumPages();
+
+ return storageProperties.getMemoryComponentPageSize() * numPages;
+ }
+ }
+
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java b/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
index 6382af9..33a7a34 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
@@ -56,7 +56,7 @@
public IIOManager getIOManager();
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int partition);
public IAsterixAppRuntimeContext getAppContext();
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index ec0d21f..b3660af 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -350,7 +350,7 @@
FileReference file = ioManager.getAbsoluteFileRef(metadataDeviceId, resourceName);
List<IVirtualBufferCache> virtualBufferCaches = runtimeContext
- .getVirtualBufferCaches(index.getDatasetId().getId());
+ .getVirtualBufferCaches(index.getDatasetId().getId(), metadataPartition.getPartitionId());
ITypeTraits[] typeTraits = index.getTypeTraits();
IBinaryComparatorFactory[] comparatorFactories = index.getKeyBinaryComparatorFactory();
int[] bloomFilterKeyFields = index.getBloomFilterKeyFields();
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
index 88e95dd..95a41c7 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
@@ -64,7 +64,7 @@
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) {
FileReference file = new FileReference(new File(filePath));
- List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
+ List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file,
runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits,
cmpFactories, bloomFilterKeyFields, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
index c087b56..cbbabbc 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
@@ -74,7 +74,7 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) throws HyracksDataException {
- List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
+ List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
try {
if (isPartitioned) {
return InvertedIndexUtils.createPartitionedLSMInvertedIndex(virtualBufferCaches,
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index 7a43849..07489ea 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -76,7 +76,7 @@
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
int partition) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
- List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
+ List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
try {
return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
index 358913b..482d1e4 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.transaction.management.service.locking;
-import static org.mockito.Mockito.mock;
-
import java.util.List;
import java.util.concurrent.Executors;
@@ -36,6 +34,8 @@
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
import org.apache.hyracks.storage.common.file.ResourceIdFactory;
+
+import static org.mockito.Mockito.mock;
class TestRuntimeContextProvider implements IAsterixAppRuntimeContextProvider {
@@ -98,7 +98,7 @@
}
@Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int partition) {
throw new UnsupportedOperationException();
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 4: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/792/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 3:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/790/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/948/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 5:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/793/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 6:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/797/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 2: Verified-1
Build Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/960/ : FAILURE
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 1:
(6 comments)
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
File asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java:
Line 40: .getVirtualBufferCaches(datasetID, ctx.getTaskAttemptId().getTaskId().getPartition());
This partition value is a task based partition (the number of partitions this task is split into which always starts from 0) and not the cluster storage partition id. For example, if you execute a query on a metadata dataset, this partition id will always be 0, whereas the storage partition for the metadata datasets could be something completely different. Therefore, this will allocate an extra VBC to this dataset and will make it exceed its limit. Similarly, in case fault tolerance is enabled, the NC will be responsible for extra storage partitions on the same dataset, and will cause extra VBCs to be allocated. I believe the best thing to do here is to pass the IO Device number. You can get it from the file split of the callers of this method as
opDesc.getFileSplitProvider().getFileSplits()[partition].getIODeviceId()
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
File asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java:
Line 746: vbcs = initializeVirtualBufferCaches(partition);
You might want to add a check here to make sure the number of VBCs of a dataset is <= numPartitions.
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
File asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java:
Line 353: .getVirtualBufferCaches(index.getDatasetId().getId(), metadataPartition.getPartitionId());
If you agree that the best thing is to pass the io device number, this needs to be changed to the IO device number of the metadataPartition.
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java:
Line 67: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
You need to pass the IO device number from RecoveryManager#startRecoveryRedoPhase of the locaResource partition. You need to add a method in PersistentLocalResourceRepository that takes a partition and return the partition IO device number on this node. (similar to PersistentLocalResourceRepository#getPartitionPath).
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java:
Line 77: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
You need to pass the IO device number from RecoveryManager#startRecoveryRedoPhase of the locaResource partition. You need to add a method in PersistentLocalResourceRepository that takes a partition and return the partition IO device number on this node. (similar to PersistentLocalResourceRepository#getPartitionPath)
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java:
Line 79: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
You need to pass the IO device number from RecoveryManager#startRecoveryRedoPhase of the locaResource partition. You need to add a method in PersistentLocalResourceRepository that takes a partition and return the partition IO device number on this node. (similar to PersistentLocalResourceRepository#getPartitionPath).
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 3: -Verified
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/791/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 6: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 3:
(9 comments)
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
File asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java:
Line 367: }
> It seems that we could remove this method from here, as the DatasetLifecycl
Done
Line 372: }
> It seems that we could remove this method from here, as the DatasetLifecycl
leaving this as-is for now, will revisit in future
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
File asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java:
Line 93: }
> It seems that we could remove this method from here, as the DatasetLifecycl
Done
Line 98: }
> It seems that we could remove this method from here, as the DatasetLifecycl
leaving this as-is for now, will revisit in the future.
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
File asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java:
Line 68: public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID);
> It seems that we could remove this method from here, as the DatasetLifecycl
Leaving this as-is for now, will revisit in the future.
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
File asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java:
Line 40: int ioDeviceNum = fileSplitProvider.getFileSplits()[ctx.getTaskAttemptId().getTaskId().getPartition()]
> Create a variable for the taskPartitionId?
Done
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
File asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java:
Line 49: public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID);
> It seems that we could remove this method from here, as the DatasetLifecycl
Leaving this as-is for now, will revisit in the future.
Line 59: public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int ioDeviceNum);
> It seems that we could remove this method from here, as the DatasetLifecycl
Done
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java:
Line 67: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, ioDeviceNum);
> Might need a variable for the DatasetLifecycleManager here, if we remove th
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Till Westmann has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 4: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 6:
@Till,
There is no need to rebase the change if there is no conflicts. It will be merged on top of the latest HEAD.
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/960/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Murtadha Hubail, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/705
to look at the new patch set (#6).
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
---
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
R asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
M asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
M asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
18 files changed, 114 insertions(+), 104 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/705/6
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 3: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/791/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 1: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/948/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 2: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Till Westmann has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 1:
(1 comment)
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
File asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java:
Line 753: int numPages = datasetID < firstAvilableUserDatasetID
s/firstAvilableUserDatasetID/firstAvailableUserDatasetID/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Murtadha Hubail, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/705
to look at the new patch set (#4).
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
---
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
R asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
M asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
M asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
18 files changed, 114 insertions(+), 104 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/705/4
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 2: -Verified
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/789/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Murtadha Hubail, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/705
to look at the new patch set (#3).
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
---
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
M asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
M asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
18 files changed, 109 insertions(+), 67 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/705/3
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 2: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/789/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 1:
(6 comments)
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
File asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java:
Line 40: .getVirtualBufferCaches(datasetID, ctx.getTaskAttemptId().getTaskId().getPartition());
> This partition value is a task based partition (the number of partitions th
Done
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
File asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java:
Line 746: vbcs = initializeVirtualBufferCaches(partition);
> You might want to add a check here to make sure the number of VBCs of a dat
Done
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
File asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java:
Line 353: .getVirtualBufferCaches(index.getDatasetId().getId(), metadataPartition.getPartitionId());
> If you agree that the best thing is to pass the io device number, this need
Done
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java:
Line 67: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
> You need to pass the IO device number from RecoveryManager#startRecoveryRed
Done
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java:
Line 77: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
> You need to pass the IO device number from RecoveryManager#startRecoveryRed
Done
https://asterix-gerrit.ics.uci.edu/#/c/705/1/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java:
Line 79: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, partition);
> You need to pass the IO device number from RecoveryManager#startRecoveryRed
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 5: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/793/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Murtadha Hubail, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/705
to look at the new patch set (#5).
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
---
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
R asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
M asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
M asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
18 files changed, 114 insertions(+), 104 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/705/5
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Till Westmann has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 3:
(10 comments)
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
File asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java:
Line 367: }
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
Line 372: }
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
File asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java:
Line 93: }
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
Line 98: }
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
File asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java:
Line 68: public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID);
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
Line 78: public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int ioDeviceNum);
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
File asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java:
Line 40: int ioDeviceNum = fileSplitProvider.getFileSplits()[ctx.getTaskAttemptId().getTaskId().getPartition()]
Create a variable for the taskPartitionId?
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
File asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java:
Line 49: public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID);
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
Line 59: public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int ioDeviceNum);
It seems that we could remove this method from here, as the DatasetLifecycleManager is accessible.
https://asterix-gerrit.ics.uci.edu/#/c/705/3/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
File asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java:
Line 67: List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID, ioDeviceNum);
Might need a variable for the DatasetLifecycleManager here, if we remove the interface methods.
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 3: Verified-1
Build Failed
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/790/ : FAILURE
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Till Westmann has submitted this change and it was merged.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/705
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hu...@gmail.com>
---
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
R asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
M asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
M asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
18 files changed, 114 insertions(+), 104 deletions(-)
Approvals:
Murtadha Hubail: Looks good to me, approved
Jenkins: Verified
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
index 5532e79..c58462f 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
@@ -21,7 +21,6 @@
import java.io.IOException;
import java.rmi.RemoteException;
import java.rmi.server.UnicastRemoteObject;
-import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -77,7 +76,6 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory;
import org.apache.hyracks.storage.common.buffercache.BufferCache;
@@ -175,7 +173,7 @@
localResourceRepository = (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory.createRepository();
- IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AsterixAppRuntimeContextProdiverForRecovery(
+ IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AsterixAppRuntimeContextProviderForRecovery(
this);
txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider,
txnProperties);
@@ -189,7 +187,8 @@
initializeResourceIdFactory();
datasetLifecycleManager = new DatasetLifecycleManager(storageProperties, localResourceRepository,
- MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager());
+ MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager(),
+ ioManager.getIODevices().size());
isShuttingdown = false;
@@ -358,11 +357,6 @@
@Override
public AsterixBuildProperties getBuildProperties() {
return buildProperties;
- }
-
- @Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
- return datasetLifecycleManager.getVirtualBufferCaches(datasetID);
}
@Override
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
similarity index 89%
rename from asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
rename to asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
index b975970..7ac5036 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.api.common;
-import java.util.List;
-
import org.apache.asterix.common.api.AsterixThreadExecutor;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
@@ -28,17 +26,16 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
-public class AsterixAppRuntimeContextProdiverForRecovery implements IAsterixAppRuntimeContextProvider {
+public class AsterixAppRuntimeContextProviderForRecovery implements IAsterixAppRuntimeContextProvider {
private final AsterixAppRuntimeContext asterixAppRuntimeContext;
- public AsterixAppRuntimeContextProdiverForRecovery(AsterixAppRuntimeContext asterixAppRuntimeContext) {
+ public AsterixAppRuntimeContextProviderForRecovery(AsterixAppRuntimeContext asterixAppRuntimeContext) {
this.asterixAppRuntimeContext = asterixAppRuntimeContext;
}
@@ -85,11 +82,6 @@
@Override
public IIOManager getIOManager() {
return asterixAppRuntimeContext.getIOManager();
- }
-
- @Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
- return asterixAppRuntimeContext.getVirtualBufferCaches(datasetID);
}
@Override
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
index 496a10e..ee9ed4a 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.rmi.RemoteException;
-import java.util.List;
import java.util.concurrent.Executor;
import org.apache.asterix.common.exceptions.ACIDException;
@@ -35,7 +34,6 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
@@ -74,8 +72,6 @@
public void deinitialize() throws HyracksDataException;
public double getBloomFilterFalsePositiveRate();
-
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
public Object getFeedManager();
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java b/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
index 3b4617a..552ce22 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
@@ -81,9 +81,10 @@
* creates (if necessary) and returns the dataset virtual buffer caches.
*
* @param datasetID
+ * @param ioDeviceNum
* @return
*/
- List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
+ List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int ioDeviceNum);
/**
* Flushes then closes all open datasets
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java b/asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
index cdd05e5..6af7441 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
@@ -27,7 +27,7 @@
public interface ILocalResourceMetadata extends Serializable {
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition) throws HyracksDataException;
+ int partition, int ioDeviceNum) throws HyracksDataException;
public int getDatasetID();
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java b/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
index c58b013..12faad2 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
@@ -22,6 +22,7 @@
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
@@ -35,9 +36,11 @@
}
@Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx) {
+ public List<IVirtualBufferCache> getVirtualBufferCaches(IHyracksTaskContext ctx, IFileSplitProvider fileSplitProvider) {
+ final int partition = ctx.getTaskAttemptId().getTaskId().getPartition();
+ final int ioDeviceNum = fileSplitProvider.getFileSplits()[partition].getIODeviceId();
return ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject())
- .getVirtualBufferCaches(datasetID);
+ .getDatasetLifecycleManager().getVirtualBufferCaches(datasetID, ioDeviceNum);
}
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index eb03015..8f9b5b0 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -53,7 +53,7 @@
public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeCycleComponent {
private final AsterixStorageProperties storageProperties;
- private final Map<Integer, List<IVirtualBufferCache>> datasetVirtualBufferCaches;
+ private final Map<Integer, DatasetVirtualBufferCaches> datasetVirtualBufferCachesMap;
private final Map<Integer, ILSMOperationTracker> datasetOpTrackers;
private final Map<Integer, DatasetInfo> datasetInfos;
private final ILocalResourceRepository resourceRepository;
@@ -62,14 +62,17 @@
private long used;
private final ILogManager logManager;
private final LogRecord logRecord;
+ private final int numPartitions;
public DatasetLifecycleManager(AsterixStorageProperties storageProperties,
- ILocalResourceRepository resourceRepository, int firstAvilableUserDatasetID, ILogManager logManager) {
+ ILocalResourceRepository resourceRepository, int firstAvilableUserDatasetID,
+ ILogManager logManager, int numPartitions) {
this.logManager = logManager;
this.storageProperties = storageProperties;
this.resourceRepository = resourceRepository;
this.firstAvilableUserDatasetID = firstAvilableUserDatasetID;
- datasetVirtualBufferCaches = new HashMap<Integer, List<IVirtualBufferCache>>();
+ this.numPartitions = numPartitions;
+ datasetVirtualBufferCachesMap = new HashMap<>();
datasetOpTrackers = new HashMap<Integer, ILSMOperationTracker>();
datasetInfos = new HashMap<Integer, DatasetInfo>();
capacity = storageProperties.getMemoryComponentGlobalBudget();
@@ -305,45 +308,36 @@
return openIndexesInfo;
}
- @Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
- synchronized (datasetVirtualBufferCaches) {
- List<IVirtualBufferCache> vbcs = datasetVirtualBufferCaches.get(datasetID);
+ private DatasetVirtualBufferCaches getVirtualBufferCaches(int datasetID) {
+ synchronized (datasetVirtualBufferCachesMap) {
+ DatasetVirtualBufferCaches vbcs = datasetVirtualBufferCachesMap.get(datasetID);
if (vbcs == null) {
- initializeDatasetVirtualBufferCache(datasetID);
- vbcs = datasetVirtualBufferCaches.get(datasetID);
- if (vbcs == null) {
- throw new IllegalStateException("Could not find dataset " + datasetID + " virtual buffer cache.");
- }
+ vbcs = initializeDatasetVirtualBufferCache(datasetID);
}
return vbcs;
}
}
+ @Override
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID, int ioDeviceNum) {
+ DatasetVirtualBufferCaches dvbcs = getVirtualBufferCaches(datasetID);
+ return dvbcs.getVirtualBufferCaches(ioDeviceNum);
+ }
+
private void removeDatasetFromCache(int datasetID) throws HyracksDataException {
deallocateDatasetMemory(datasetID);
datasetInfos.remove(datasetID);
- datasetVirtualBufferCaches.remove(datasetID);
+ datasetVirtualBufferCachesMap.remove(datasetID);
datasetOpTrackers.remove(datasetID);
}
- private void initializeDatasetVirtualBufferCache(int datasetID) {
- List<IVirtualBufferCache> vbcs = new ArrayList<IVirtualBufferCache>();
- synchronized (datasetVirtualBufferCaches) {
- int numPages = datasetID < firstAvilableUserDatasetID
- ? storageProperties.getMetadataMemoryComponentNumPages()
- : storageProperties.getMemoryComponentNumPages();
- for (int i = 0; i < storageProperties.getMemoryComponentsNum(); i++) {
- MultitenantVirtualBufferCache vbc = new MultitenantVirtualBufferCache(
- new VirtualBufferCache(new ResourceHeapBufferAllocator(this, Integer.toString(datasetID)),
- storageProperties.getMemoryComponentPageSize(),
- numPages / storageProperties.getMemoryComponentsNum()));
- vbcs.add(vbc);
- }
- datasetVirtualBufferCaches.put(datasetID, vbcs);
+ private DatasetVirtualBufferCaches initializeDatasetVirtualBufferCache(int datasetID) {
+ synchronized (datasetVirtualBufferCachesMap) {
+ DatasetVirtualBufferCaches dvbcs = new DatasetVirtualBufferCaches(datasetID);
+ datasetVirtualBufferCachesMap.put(datasetID, dvbcs);
+ return dvbcs;
}
}
-
@Override
public ILSMOperationTracker getOperationTracker(int datasetID) {
synchronized (datasetOpTrackers) {
@@ -356,7 +350,7 @@
}
}
- private abstract class Info {
+ private static abstract class Info {
protected int referenceCount;
protected boolean isOpen;
@@ -374,7 +368,7 @@
}
}
- public class IndexInfo extends Info {
+ public static class IndexInfo extends Info {
private final ILSMIndex index;
private final long resourceId;
private final int datasetId;
@@ -398,7 +392,7 @@
}
}
- public class DatasetInfo extends Info implements Comparable<DatasetInfo> {
+ public static class DatasetInfo extends Info implements Comparable<DatasetInfo> {
private final Map<Long, IndexInfo> indexes;
private final int datasetID;
private long lastAccess;
@@ -639,7 +633,7 @@
closeAllDatasets();
- datasetVirtualBufferCaches.clear();
+ datasetVirtualBufferCachesMap.clear();
datasetOpTrackers.clear();
datasetInfos.clear();
}
@@ -687,11 +681,7 @@
synchronized (dsInfo) {
// This is not needed for external datasets' indexes since they never use the virtual buffer cache.
if (!dsInfo.memoryAllocated && !dsInfo.isExternal) {
- List<IVirtualBufferCache> vbcs = getVirtualBufferCaches(dsInfo.datasetID);
- long additionalSize = 0;
- for (IVirtualBufferCache vbc : vbcs) {
- additionalSize += vbc.getNumPages() * vbc.getPageSize();
- }
+ long additionalSize = getVirtualBufferCaches(dsInfo.datasetID).getTotalSize();
while (used + additionalSize > capacity) {
if (!evictCandidateDataset()) {
throw new HyracksDataException("Cannot allocate dataset " + dsInfo.datasetID
@@ -712,10 +702,7 @@
}
synchronized (dsInfo) {
if (dsInfo.isOpen && dsInfo.memoryAllocated) {
- List<IVirtualBufferCache> vbcs = getVirtualBufferCaches(dsInfo.datasetID);
- for (IVirtualBufferCache vbc : vbcs) {
- used -= (vbc.getNumPages() * vbc.getPageSize());
- }
+ used -= getVirtualBufferCaches(dsInfo.datasetID).getTotalSize();
dsInfo.memoryAllocated = false;
}
}
@@ -727,4 +714,49 @@
int did = Integer.parseInt(resourcePath);
allocateDatasetMemory(did);
}
+
+ private class DatasetVirtualBufferCaches {
+ private final int datasetID;
+ private final Map<Integer, List<IVirtualBufferCache>> ioDeviceVirtualBufferCaches = new HashMap<>();
+
+ public DatasetVirtualBufferCaches(int datasetID) {
+ this.datasetID = datasetID;
+ }
+
+ private List<IVirtualBufferCache> initializeVirtualBufferCaches(int ioDeviceNum) {
+ assert ioDeviceVirtualBufferCaches.size() < numPartitions;
+ int numPages = datasetID < firstAvilableUserDatasetID
+ ? storageProperties.getMetadataMemoryComponentNumPages()
+ : storageProperties.getMemoryComponentNumPages();
+ List<IVirtualBufferCache> vbcs = new ArrayList<>();
+ for (int i = 0; i < storageProperties.getMemoryComponentsNum(); i++) {
+ MultitenantVirtualBufferCache vbc = new MultitenantVirtualBufferCache(
+ new VirtualBufferCache(new ResourceHeapBufferAllocator(DatasetLifecycleManager.this,
+ Integer.toString(datasetID)), storageProperties.getMemoryComponentPageSize(),
+ numPages / storageProperties.getMemoryComponentsNum() / numPartitions));
+ vbcs.add(vbc);
+ }
+ ioDeviceVirtualBufferCaches.put(ioDeviceNum, vbcs);
+ return vbcs;
+ }
+
+ public List<IVirtualBufferCache> getVirtualBufferCaches(int ioDeviceNum) {
+ synchronized (ioDeviceVirtualBufferCaches) {
+ List<IVirtualBufferCache> vbcs = ioDeviceVirtualBufferCaches.get(ioDeviceNum);
+ if (vbcs == null) {
+ vbcs = initializeVirtualBufferCaches(ioDeviceNum);
+ }
+ return vbcs;
+ }
+ }
+
+ public long getTotalSize() {
+ int numPages = datasetID < firstAvilableUserDatasetID
+ ? storageProperties.getMetadataMemoryComponentNumPages()
+ : storageProperties.getMemoryComponentNumPages();
+
+ return storageProperties.getMemoryComponentPageSize() * numPages;
+ }
+ }
+
}
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java b/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
index 6382af9..d0dd3fa 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
@@ -18,15 +18,12 @@
*/
package org.apache.asterix.common.transactions;
-import java.util.List;
-
import org.apache.asterix.common.api.AsterixThreadExecutor;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
@@ -55,8 +52,6 @@
public IResourceIdFactory getResourceIdFactory();
public IIOManager getIOManager();
-
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID);
public IAsterixAppRuntimeContext getAppContext();
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index b83f2f3..07d0364 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -349,8 +349,8 @@
String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath();
FileReference file = ioManager.getAbsoluteFileRef(metadataDeviceId, resourceName);
- List<IVirtualBufferCache> virtualBufferCaches = runtimeContext
- .getVirtualBufferCaches(index.getDatasetId().getId());
+ List<IVirtualBufferCache> virtualBufferCaches = runtimeContext.getDatasetLifecycleManager()
+ .getVirtualBufferCaches(index.getDatasetId().getId(), metadataPartition.getIODeviceNum());
ITypeTraits[] typeTraits = index.getTypeTraits();
IBinaryComparatorFactory[] comparatorFactories = index.getKeyBinaryComparatorFactory();
int[] bloomFilterKeyFields = index.getBloomFilterKeyFields();
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
index a75428f..33b7f30 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
@@ -45,7 +45,7 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition) {
+ int partition, int ioDeviceNum) {
FileReference file = new FileReference(new File(filePath));
LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields,
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
index 7b487d4..04006f1 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
@@ -58,7 +58,7 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition) throws HyracksDataException {
+ int partition, int ioDeviceNum) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
return LSMBTreeUtils.createExternalBTreeWithBuddy(file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, btreeCmpFactories,
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
index 89a5165..ed468b9 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
@@ -54,7 +54,7 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition) throws HyracksDataException {
+ int partition, int ioDeviceNum) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
try {
return LSMRTreeUtils.createExternalRTree(file, runtimeContextProvider.getBufferCache(),
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
index 88e95dd..b1db1f3 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
@@ -19,9 +19,9 @@
package org.apache.asterix.transaction.management.resource;
import java.io.File;
-import java.util.List;
import java.util.Map;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
@@ -32,7 +32,6 @@
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
public class LSMBTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetadata {
@@ -62,17 +61,15 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition) {
+ int partition, int ioDeviceNum) {
FileReference file = new FileReference(new File(filePath));
- List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
- LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file,
- runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(), typeTraits,
- cmpFactories, bloomFilterKeyFields, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
- mergePolicyFactory.createMergePolicy(mergePolicyProperties,
- runtimeContextProvider.getDatasetLifecycleManager()),
+ final IDatasetLifecycleManager datasetLifecycleManager = runtimeContextProvider.getDatasetLifecycleManager();
+ LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(datasetLifecycleManager.getVirtualBufferCaches(datasetID,
+ ioDeviceNum), file, runtimeContextProvider.getBufferCache(), runtimeContextProvider.getFileMapManager(),
+ typeTraits, cmpFactories, bloomFilterKeyFields, runtimeContextProvider.getBloomFilterFalsePositiveRate(),
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties, datasetLifecycleManager),
isPrimary ? runtimeContextProvider.getLSMBTreeOperationTracker(datasetID)
- : new BaseOperationTracker(datasetID,
- runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
+ : new BaseOperationTracker(datasetID, datasetLifecycleManager.getDatasetInfo(datasetID)),
runtimeContextProvider.getLSMIOScheduler(),
LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), isPrimary, filterTypeTraits,
filterCmpFactories, btreeFields, filterFields, true);
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
index c087b56..87319b4 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
@@ -73,8 +73,9 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition) throws HyracksDataException {
- List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
+ int partition, int ioDeviceNum) throws HyracksDataException {
+ List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager()
+ .getVirtualBufferCaches(datasetID, ioDeviceNum);
try {
if (isPartitioned) {
return InvertedIndexUtils.createPartitionedLSMInvertedIndex(virtualBufferCaches,
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index 7a43849..bae36d4 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -74,9 +74,10 @@
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition) throws HyracksDataException {
+ int partition, int ioDeviceNum) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
- List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getVirtualBufferCaches(datasetID);
+ List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager()
+ .getVirtualBufferCaches(datasetID, ioDeviceNum);
try {
return LSMRTreeUtils.createLSMTree(virtualBufferCaches, file, runtimeContextProvider.getBufferCache(),
runtimeContextProvider.getFileMapManager(), typeTraits, rtreeCmpFactories, btreeCmpFactories,
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index db7c0a0..3a1e729 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -419,7 +419,11 @@
*/
public String getPartitionPath(int partition) {
//currently each partition is replicated on the same IO device number on all NCs.
- return mountPoints[clusterPartitions.get(partition).getIODeviceNum()];
+ return mountPoints[getIODeviceNum(partition)];
+ }
+
+ public int getIODeviceNum(int partition) {
+ return clusterPartitions.get(partition).getIODeviceNum();
}
public Set<Integer> getActivePartitions() {
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
index 358913b..3a4a56a 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
@@ -18,9 +18,6 @@
*/
package org.apache.asterix.transaction.management.service.locking;
-import static org.mockito.Mockito.mock;
-
-import java.util.List;
import java.util.concurrent.Executors;
import org.apache.asterix.common.api.AsterixThreadExecutor;
@@ -31,11 +28,12 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
import org.apache.hyracks.storage.common.file.ResourceIdFactory;
+
+import static org.mockito.Mockito.mock;
class TestRuntimeContextProvider implements IAsterixAppRuntimeContextProvider {
@@ -94,11 +92,6 @@
@Override
public IIOManager getIOManager() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<IVirtualBufferCache> getVirtualBufferCaches(int datasetID) {
throw new UnsupportedOperationException();
}
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
index c462b22..9a74a18 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
@@ -370,7 +370,8 @@
//#. create index instance and register to indexLifeCycleManager
localResourceMetadata = (ILocalResourceMetadata) localResource.getResourceObject();
index = localResourceMetadata.createIndexInstance(appRuntimeContext,
- resourceAbsolutePath, localResource.getPartition());
+ resourceAbsolutePath, localResource.getPartition(),
+ localResourceRepository.getIODeviceNum(localResource.getPartition()));
datasetLifecycleManager.register(resourceAbsolutePath, index);
datasetLifecycleManager.open(resourceAbsolutePath);
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 6: Verified+1
Build Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/797/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/705
to look at the new patch set (#2).
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
As sugggested in ASTERIXDB-1337, this change is to maintain a per-
partition MultitenantVirtualBufferCache budget, as opposed to sharing
the budget across the dataset.
Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
---
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
M asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProdiverForRecovery.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
M asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/api/ILocalResourceMetadata.java
M asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java
M asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M asterix-common/src/main/java/org/apache/asterix/common/transactions/IAsterixAppRuntimeContextProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeWithBuddyLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/TestRuntimeContextProvider.java
M asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/RecoveryManager.java
18 files changed, 109 insertions(+), 67 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/705/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 4:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/792/
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: ASTERIXDB-1337: Dataset Memory Management on Multi-Partition NC
......................................................................
Patch Set 5: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/705
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: Ibbf08f532c1210c30be6a51c73570a789174213b
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No