You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2018/02/07 16:43:38 UTC
asterixdb git commit: [ASTERIXDB-2276][CONF] Introduce Max Active
Writable Datasets
Repository: asterixdb
Updated Branches:
refs/heads/master 2530e39d7 -> f4485553d
[ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Introduce number of metadata datasets parameter
and use it to reserve memory for all metadata
datasets.
- Default metadata dataset memory component number
of pages to 32 pages.
- Introduce max active writable datasets and use
it to automatically calculate the number of pages
of a user dataset memory component.
- Remve the assumpation of reserving an NC core
for heartbeats.
- Fix for [ASTERIXDB-2279] by making the expected
result order deterministic.
Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2362
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f4485553
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f4485553
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f4485553
Branch: refs/heads/master
Commit: f4485553da14fff7c653c7724cb8b8256ada7b3b
Parents: 2530e39
Author: Murtadha Hubail <mh...@apache.org>
Authored: Wed Feb 7 16:33:13 2018 +0300
Committer: Murtadha Hubail <mh...@apache.org>
Committed: Wed Feb 7 08:43:09 2018 -0800
----------------------------------------------------------------------
.../hyracks/bootstrap/NCApplication.java | 14 +---
.../asterix-app/src/main/resources/cc.conf | 1 -
.../asterix-app/src/main/resources/cc2.conf | 1 -
.../asterix-app/src/main/resources/cc3.conf | 1 -
.../ClusterStateDefaultParameterTest.java | 4 +-
.../asterix/test/txn/RecoveryManagerTest.java | 5 +-
.../src/test/resources/cc-multipart.conf | 1 -
.../resources/cc-small-txn-log-partition.conf | 6 +-
.../asterix-app/src/test/resources/cc.conf | 1 -
.../query-ASTERIXDB-971.3.query.sqlpp | 3 +-
.../cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full.1.regexadm | 1 +
.../cluster_state_1_less.1.regexadm | 1 +
.../common/config/StorageProperties.java | 72 ++++++++++++--------
14 files changed, 56 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 1220462..b0382f7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -222,19 +222,9 @@ public class NCApplication extends BaseNCApplication {
@Override
public NodeCapacity getCapacity() {
StorageProperties storageProperties = runtimeContext.getStorageProperties();
- // Deducts the reserved buffer cache size and memory component size from the maxium heap size,
- // and deducts one core for processing heartbeats.
- long memorySize = Runtime.getRuntime().maxMemory() - storageProperties.getBufferCacheSize()
- - storageProperties.getMemoryComponentGlobalBudget();
- if (memorySize <= 0) {
- throw new IllegalStateException("Invalid node memory configuration, more memory budgeted than available "
- + "in JVM. Runtime max memory: " + Runtime.getRuntime().maxMemory() + " Buffer cache size: "
- + storageProperties.getBufferCacheSize() + " Memory component global budget: "
- + storageProperties.getMemoryComponentGlobalBudget());
- }
+ final long memorySize = storageProperties.getJobExecutionMemoryBudget();
int allCores = Runtime.getRuntime().availableProcessors();
- int maximumCoresForComputation = allCores > 1 ? allCores - 1 : allCores;
- return new NodeCapacity(memorySize, maximumCoresForComputation);
+ return new NodeCapacity(memorySize, allCores);
}
private void performLocalCleanUp() throws HyracksDataException {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/main/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index 914b7b6..fe6cf64 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -37,7 +37,6 @@ app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/main/resources/cc2.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index 6c01386..113497d 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -37,7 +37,6 @@ app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/main/resources/cc3.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index 933e4af..e210bf62 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -37,7 +37,6 @@ app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
index f029669..e752493 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
@@ -87,8 +87,8 @@ public class ClusterStateDefaultParameterTest {
Assert.assertTrue(getValue(row) == maxHeap / 4);
matchCount++;
}
- if (row.contains("storage.memorycomponent.numpages")) {
- Assert.assertTrue(getValue(row) == maxHeap / (131072 * 64));
+ if (row.contains("storage.max.active.writable.datasets")) {
+ Assert.assertTrue(getValue(row) == 8);
matchCount++;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
index 92e403c..05e5aad 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
@@ -48,10 +48,7 @@ public class RecoveryManagerTest {
@Before
public void setUp() throws Exception {
- // Read default test configurations
- // Write test config file
- integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET, "128MB");
- integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_NUMPAGES, 32);
+ integrationUtil.addOption(StorageProperties.Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS, 20);
integrationUtil.setGracefulShutdown(false);
integrationUtil.init(true, TEST_CONFIG_FILE_PATH);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
index f28545d..5cf1bbe 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
@@ -38,7 +38,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyRe
storage.subdir=test_storage
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=32
storage.memorycomponent.globalbudget=512MB
[cc]
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
index 0b66cfe..6cd3b87 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
@@ -35,8 +35,7 @@ app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
-storage.memorycomponent.globalbudget=512MB
+storage.memorycomponent.globalbudget=128MB
[cc]
address = 127.0.0.1
@@ -53,4 +52,5 @@ compiler.joinmemory=256KB
messaging.frame.size=4096
messaging.frame.count=512
txn.log.partitionsize=2MB
-txn.log.checkpoint.pollfrequency=2147483647
\ No newline at end of file
+txn.log.checkpoint.pollfrequency=2147483647
+storage.max.active.writable.datasets=50
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/resources/cc.conf
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index 3e4ecd9..5cf1bbe 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -38,7 +38,6 @@ jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyRe
storage.subdir=test_storage
storage.buffercache.pagesize=32KB
storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
storage.memorycomponent.globalbudget=512MB
[cc]
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
index 6856726..cd1bcfc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
@@ -23,4 +23,5 @@ FROM range(1,2) AS x
SELECT ELEMENT (
WITH z AS (FROM Accounts AS y WHERE y.id=x SELECT VALUE y)
SELECT x, z
-);
+)
+ORDER BY x;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 85b121e..59057b1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -28,6 +28,7 @@
"replication\.log\.buffer\.pagesize" : 131072,
"replication\.strategy" : "none",
"replication\.timeout" : 30,
+ "storage.max.active.writable.datasets" : 8,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
"txn\.job\.recovery\.memorysize" : 67108864,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 7017b06..13dacb8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -28,6 +28,7 @@
"replication\.log\.buffer\.pagesize" : 131072,
"replication\.strategy" : "none",
"replication\.timeout" : 30,
+ "storage.max.active.writable.datasets" : 8,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
"txn\.job\.recovery\.memorysize" : 67108864,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 00e4d67..9f18638 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -28,6 +28,7 @@
"replication\.log\.buffer\.pagesize" : 131072,
"replication\.strategy" : "none",
"replication\.timeout" : 30,
+ "storage.max.active.writable.datasets" : 8,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
"txn\.job\.recovery\.memorysize" : 67108864,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4485553/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index 963f0ca..e01198f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -42,16 +42,11 @@ public class StorageProperties extends AbstractProperties {
STORAGE_BUFFERCACHE_MAXOPENFILES(INTEGER, Integer.MAX_VALUE),
STORAGE_MEMORYCOMPONENT_GLOBALBUDGET(LONG_BYTE_UNIT, Runtime.getRuntime().maxMemory() / 4),
STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)),
- STORAGE_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig, Integer>) accessor ->
- // By default, uses 1/16 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET for the write buffer
- // budget for a dataset, including data and indexes.
- (int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (16 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE)))),
STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(INTEGER, 2),
- STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig, Integer>) accessor ->
- // By default, uses the min of 1/64 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET and 256 pages
- // for the write buffer budget for a metadata dataset, including data and indexes.
- Math.min((int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (64 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE))), 256)),
- STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d);
+ STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, 32),
+ STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d),
+ STORAGE_METADATA_DATASETS(INTEGER, 14),
+ STORAGE_MAX_ACTIVE_WRITABLE_DATASETS(INTEGER, 8);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -61,13 +56,11 @@ public class StorageProperties extends AbstractProperties {
this.defaultValue = defaultValue;
}
- <T> Option(IOptionType<T> interpreter, Function<IApplicationConfig, T> defaultValueFunction) {
- this.interpreter = interpreter;
- this.defaultValue = defaultValueFunction;
- }
-
@Override
public Section section() {
+ if (this == STORAGE_MAX_ACTIVE_WRITABLE_DATASETS) {
+ return Section.COMMON;
+ }
return Section.NC;
}
@@ -86,17 +79,16 @@ public class StorageProperties extends AbstractProperties {
+ "of the memory component page size";
case STORAGE_MEMORYCOMPONENT_PAGESIZE:
return "The page size in bytes for pages allocated to memory components";
- case STORAGE_MEMORYCOMPONENT_NUMPAGES:
- return "The number of pages to allocate for a memory component. This budget is shared by all "
- + "the memory components of the primary index and all its secondary indexes across all I/O "
- + "devices on a node. Note: in-memory components usually has fill factor of 75% since "
- + "the pages are 75% full and the remaining 25% is un-utilized";
case STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS:
return "The number of memory components to be used per lsm index";
case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES:
return "The number of pages to allocate for a metadata memory component";
case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE:
return "The maximum acceptable false positive rate for bloom filters associated with LSM indexes";
+ case STORAGE_METADATA_DATASETS:
+ return "The number of metadata datasets";
+ case STORAGE_MAX_ACTIVE_WRITABLE_DATASETS:
+ return "The maximum number of datasets that can be concurrently modified";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -114,16 +106,10 @@ public class StorageProperties extends AbstractProperties {
@Override
public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) {
- switch (this) {
- case STORAGE_MEMORYCOMPONENT_NUMPAGES:
- return "1/16th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET)
- + " value";
- case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES:
- return "1/64th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET)
- + " value or 256, whichever is larger";
- default:
- return null;
+ if (this == STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES) {
+ return "32 pages";
}
+ return null;
}
}
@@ -148,7 +134,10 @@ public class StorageProperties extends AbstractProperties {
}
public int getMemoryComponentNumPages() {
- return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_NUMPAGES);
+ final long metadataReservedMem = getMetadataReservedMemory();
+ final long globalUserDatasetMem = getMemoryComponentGlobalBudget() - metadataReservedMem;
+ final long userDatasetMem = globalUserDatasetMem / getMaxActiveWritableDatasets();
+ return (int) (userDatasetMem / getMemoryComponentPageSize());
}
public int getMetadataMemoryComponentNumPages() {
@@ -170,4 +159,29 @@ public class StorageProperties extends AbstractProperties {
public int getBufferCacheNumPages() {
return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES));
}
+
+ public long getJobExecutionMemoryBudget() {
+ final long jobExecutionMemory =
+ Runtime.getRuntime().maxMemory() - getBufferCacheSize() - getMemoryComponentGlobalBudget();
+ if (jobExecutionMemory <= 0) {
+ final String msg = String.format(
+ "Invalid node memory configuration, more memory budgeted than available in JVM. Runtime max memory:"
+ + " (%d), Buffer cache memory (%d), memory component global budget (%d)",
+ Runtime.getRuntime().maxMemory(), getBufferCacheSize(), getMemoryComponentGlobalBudget());
+ throw new IllegalStateException(msg);
+ }
+ return jobExecutionMemory;
+ }
+
+ public int getMaxActiveWritableDatasets() {
+ return accessor.getInt(Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS);
+ }
+
+ private int getMetadataDatasets() {
+ return accessor.getInt(Option.STORAGE_METADATA_DATASETS);
+ }
+
+ private long getMetadataReservedMemory() {
+ return (getMetadataMemoryComponentNumPages() * (long) getMemoryComponentPageSize()) * getMetadataDatasets();
+ }
}