You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2020/12/23 22:35:19 UTC
[asterixdb] branch master updated (59e3d6a -> 0b0d19a)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 59e3d6a Merge branch 'gerrit/cheshire-cat'
new 8668d2e [NO ISSUE][*DB] Lock manager allocation improvements
new 4e00bb4 [NO ISSUE][HYR] Logging improvements, cleanup
new 92ae16e [NO ISSUE][HYR][UTIL] += Span.ELAPSED; misc cleanup
new 0b0d19a Merge branch 'gerrit/cheshire-cat'
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../asterix-app/src/main/resources/cc-rep.conf | 3 +-
asterixdb/asterix-app/src/main/resources/cc.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc2.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc3.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc4.conf | 1 -
.../src/test/resources/cc-compression.conf | 1 -
.../src/test/resources/cc-multipart.conf | 1 -
.../asterix-app/src/test/resources/cc-rep-ssl.conf | 3 +-
.../asterix-app/src/test/resources/cc-single.conf | 1 -
.../test/resources/cc-small-txn-log-partition.conf | 1 -
.../asterix-app/src/test/resources/cc-ssl.conf | 3 +-
.../asterix-app/src/test/resources/cc-storage.conf | 3 +-
asterixdb/asterix-app/src/test/resources/cc.conf | 1 -
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 +-
.../cluster_state_1_full.1.regexadm | 2 +-
.../cluster_state_1_less.1.regexadm | 2 +-
.../asterix/common/config/StorageProperties.java | 10 ++---
.../common/config/TransactionProperties.java | 24 +++++++++++-
.../service/locking/ConcurrentLockManager.java | 20 +++++-----
.../management/service/locking/ResourceGroup.java | 25 +++++++------
.../service/locking/ResourceGroupTable.java | 10 ++---
.../service/locking/ResourceTablePrinter.java | 4 +-
.../org/apache/hyracks/api/dataflow/IActivity.java | 22 +++++++++--
.../runtime/SuperActivityOperatorNodePushable.java | 4 +-
.../hyracks/api/dataflow/IActivityTest.java} | 43 ++++++++++++----------
.../hyracks/control/nc/work/StartTasksWork.java | 5 +--
.../main/java/org/apache/hyracks/util/Span.java | 14 ++-----
27 files changed, 112 insertions(+), 96 deletions(-)
copy hyracks-fullstack/hyracks/{hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java => hyracks-api/src/test/java/org/apache/hyracks/api/dataflow/IActivityTest.java} (53%)
[asterixdb] 02/04: [NO ISSUE][HYR] Logging improvements, cleanup
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4e00bb4970f88a1285d4a623ae04feef6876fe8c
Author: Michael Blow <mb...@apache.org>
AuthorDate: Sun Dec 20 11:35:58 2020 -0500
[NO ISSUE][HYR] Logging improvements, cleanup
- abbreviate java package names in IActivity display names, as these
are used in thread names, which adds a lot of bloat to logs: all
but the last segment of the package are replaced with the first
letter of the segment
e.g. org.apache.hyracks.api.rewriter.runtime.SuperActivity
-> o.a.h.a.r.runtime.SuperActivity
Change-Id: I092f2988f1efa0ffbf7e6395313edbc482f377a4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9403
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../org/apache/hyracks/api/dataflow/IActivity.java | 22 +++++++++++++---
.../runtime/SuperActivityOperatorNodePushable.java | 4 +--
.../hyracks/api/dataflow/IActivityTest.java} | 30 +++++++++++++++++++---
.../hyracks/control/nc/work/StartTasksWork.java | 5 ++--
4 files changed, 49 insertions(+), 12 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IActivity.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IActivity.java
index f7e4d42..033e744 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IActivity.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IActivity.java
@@ -19,14 +19,30 @@
package org.apache.hyracks.api.dataflow;
import java.io.Serializable;
+import java.util.regex.Pattern;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IActivity extends Serializable {
- public ActivityId getActivityId();
+ ActivityId getActivityId();
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException;
+ IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
+ int partition, int nPartitions) throws HyracksDataException;
+
+ default String getDisplayName() {
+ return DisplayNameHelper.toDisplayName(getClass().getName());
+ }
+
+ class DisplayNameHelper {
+ static final Pattern PREFIX_PATTERN = Pattern.compile("\\B\\w+(\\.[a-z])");
+
+ private DisplayNameHelper() {
+ }
+
+ static String toDisplayName(String className) {
+ return PREFIX_PATTERN.matcher(className).replaceAll("$1");
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
index 168d7dd..1d8c774 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
@@ -55,6 +55,7 @@ import org.apache.hyracks.api.util.ExceptionUtils;
* connected activities in a single thread.
*/
public class SuperActivityOperatorNodePushable implements IOperatorNodePushable {
+ private static final String CLASS_SIMPLE_NAME = SuperActivityOperatorNodePushable.class.getSimpleName();
private final Map<ActivityId, IOperatorNodePushable> operatorNodePushables = new HashMap<>();
private final List<IOperatorNodePushable> operatorNodePushablesBFSOrder = new ArrayList<>();
private final Map<ActivityId, IActivity> startActivities;
@@ -223,8 +224,7 @@ public class SuperActivityOperatorNodePushable implements IOperatorNodePushable
try {
Thread.currentThread()
.setName(Thread.currentThread().getName() + ":" + ctx.getJobletContext().getJobId()
- + ":" + ctx.getTaskAttemptId() + ":"
- + SuperActivityOperatorNodePushable.class.getSimpleName());
+ + ":" + ctx.getTaskAttemptId() + ":" + CLASS_SIMPLE_NAME);
action.run(op);
} catch (Throwable th) { // NOSONAR: Must catch all causes of failure
failures.offer(th);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IActivity.java b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/dataflow/IActivityTest.java
similarity index 52%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IActivity.java
copy to hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/dataflow/IActivityTest.java
index f7e4d42..98830c4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IActivity.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/dataflow/IActivityTest.java
@@ -18,15 +18,37 @@
*/
package org.apache.hyracks.api.dataflow;
-import java.io.Serializable;
+import static org.apache.hyracks.api.dataflow.IActivity.DisplayNameHelper.toDisplayName;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.rewriter.runtime.SuperActivity;
+import org.junit.Assert;
+import org.junit.Test;
-public interface IActivity extends Serializable {
- public ActivityId getActivityId();
+public class IActivityTest {
+ @Test
+ public void testDisplayName() {
+ IActivity dummyActivity = new DummyActivity();
+ final String displayName = dummyActivity.getDisplayName();
+ Assert.assertEquals("o.a.h.a.dataflow.DummyActivity", displayName);
+ Assert.assertEquals("o.a.h.a.r.runtime.SuperActivity", toDisplayName(SuperActivity.class.getName()));
+ Assert.assertEquals("o.a.h.a.r.o.std.SplitOperatorDescriptor$AbstractReplicateOperatorDescriptor",
+ toDisplayName("org.apache.hyracks.algebricks.runtime.operators.std.SplitOperatorDescriptor"
+ + "$AbstractReplicateOperatorDescriptor"));
+ }
+}
+
+class DummyActivity implements IActivity {
+ @Override
+ public ActivityId getActivityId() {
+ return null;
+ }
+ @Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException;
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return null;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java
index 69f1113..551d8fa 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java
@@ -150,8 +150,7 @@ public class StartTasksWork extends AbstractWork {
LOGGER.trace("Initializing {} -> {} for {}", taId, han, jobId);
final int partition = tid.getPartition();
List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(aid);
- task = null;
- task = new Task(joblet, flags, taId, han.getClass().getName(), ncs.getExecutor(), ncs,
+ task = new Task(joblet, flags, taId, han.getDisplayName(), ncs.getExecutor(), ncs,
createInputChannels(td, inputs));
IOperatorNodePushable operator = han.createPushRuntime(task, rdp, partition, td.getPartitionCount());
List<IPartitionCollector> collectors = new ArrayList<>();
@@ -185,7 +184,7 @@ public class StartTasksWork extends AbstractWork {
}
}
- task.setTaskRuntime(collectors.toArray(new IPartitionCollector[collectors.size()]), operator);
+ task.setTaskRuntime(collectors.toArray(new IPartitionCollector[0]), operator);
joblet.addTask(task);
task.start();
taskIndex++;
[asterixdb] 01/04: [NO ISSUE][*DB] Lock manager allocation
improvements
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8668d2eceda1287c4fc45b7d70a35bd6fea2ca08
Author: Michael Blow <mb...@apache.org>
AuthorDate: Fri Dec 18 17:04:41 2020 -0500
[NO ISSUE][*DB] Lock manager allocation improvements
1. default lock table size to 1009 for -Xmx < 8GB, otherwise 1048583
2. reduce memory footprint of lock table entries
a. favor volatile over AtomicLong for ResourceGroup.firstResourceIndex
b. replace ReentrantReadWriteLock w/ ReentrantLock as RW is not used
Change-Id: I6d2e8b3e2bbd1783171c9372a9570b99a846b72d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9363
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Luo Chen <cl...@uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../asterix-app/src/main/resources/cc-rep.conf | 3 +--
asterixdb/asterix-app/src/main/resources/cc.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc2.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc3.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc4.conf | 1 -
.../src/test/resources/cc-compression.conf | 1 -
.../src/test/resources/cc-multipart.conf | 1 -
.../asterix-app/src/test/resources/cc-rep-ssl.conf | 3 +--
.../asterix-app/src/test/resources/cc-single.conf | 1 -
.../test/resources/cc-small-txn-log-partition.conf | 1 -
.../asterix-app/src/test/resources/cc-ssl.conf | 3 +--
.../asterix-app/src/test/resources/cc-storage.conf | 3 +--
asterixdb/asterix-app/src/test/resources/cc.conf | 1 -
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 +-
.../cluster_state_1_full.1.regexadm | 2 +-
.../cluster_state_1_less.1.regexadm | 2 +-
.../asterix/common/config/StorageProperties.java | 10 ++++-----
.../common/config/TransactionProperties.java | 24 ++++++++++++++++++++-
.../service/locking/ConcurrentLockManager.java | 20 ++++++++---------
.../management/service/locking/ResourceGroup.java | 25 +++++++++++-----------
.../service/locking/ResourceGroupTable.java | 10 +++------
.../service/locking/ResourceTablePrinter.java | 4 ++--
22 files changed, 63 insertions(+), 57 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/resources/cc-rep.conf b/asterixdb/asterix-app/src/main/resources/cc-rep.conf
index 6e20dd7..818b835 100644
--- a/asterixdb/asterix-app/src/main/resources/cc-rep.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc-rep.conf
@@ -48,5 +48,4 @@ heartbeat.max.misses=25
[common]
log.level = INFO
replication.enabled=true
-replication.strategy=all
-txn.lock.table.size=1000
\ No newline at end of file
+replication.strategy=all
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index e8f9897..51c71c2 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -62,4 +62,3 @@ compiler.sort.parallel=false
compiler.internal.sanitycheck=true
messaging.frame.size=4096
messaging.frame.count=512
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index e5da5e3..016c485 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -57,4 +57,3 @@ compiler.windowmemory=192KB
compiler.parallelism=-1
messaging.frame.size=4096
messaging.frame.count=512
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index 121ed85..88362aa 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -57,4 +57,3 @@ compiler.windowmemory=192KB
compiler.parallelism=3
messaging.frame.size=4096
messaging.frame.count=512
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/main/resources/cc4.conf b/asterixdb/asterix-app/src/main/resources/cc4.conf
index f5025a5..6a66d25 100644
--- a/asterixdb/asterix-app/src/main/resources/cc4.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc4.conf
@@ -54,4 +54,3 @@ compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
compiler.parallelism=-1
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-compression.conf b/asterixdb/asterix-app/src/test/resources/cc-compression.conf
index bd7f1d0..c8d9780 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-compression.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-compression.conf
@@ -56,4 +56,3 @@ compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
storage.compression.block=snappy
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
index b33221d..5543e51 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
@@ -54,4 +54,3 @@ compiler.joinmemory=256KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf b/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf
index 7146c22..db4ca20 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-rep-ssl.conf
@@ -58,5 +58,4 @@ trust.store.path=security/root/root.truststore
log.level = INFO
replication.enabled=true
replication.strategy=all
-ssl.enabled=true
-txn.lock.table.size=1000
\ No newline at end of file
+ssl.enabled=true
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single.conf b/asterixdb/asterix-app/src/test/resources/cc-single.conf
index 875af51..794b5b0 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single.conf
@@ -48,4 +48,3 @@ compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
index 4091625..dd8edb4 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
@@ -56,4 +56,3 @@ txn.log.partitionsize=2MB
txn.log.buffer.pagesize=128KB
txn.log.checkpoint.pollfrequency=2147483647
txn.log.checkpoint.history=0
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
index 47171e0..499e9fc 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
@@ -67,5 +67,4 @@ compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
-ssl.enabled=true
-txn.lock.table.size=1000
\ No newline at end of file
+ssl.enabled=true
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc-storage.conf b/asterixdb/asterix-app/src/test/resources/cc-storage.conf
index bc50c13..9e869c7 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-storage.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-storage.conf
@@ -55,5 +55,4 @@ messaging.frame.size=4096
messaging.frame.count=512
storage.compression.block=snappy
txn.log.checkpoint.pollfrequency=10
-txn.dataset.checkpoint.interval=10
-txn.lock.table.size=1000
\ No newline at end of file
+txn.dataset.checkpoint.interval=10
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index 51ee756..1899122 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -59,4 +59,3 @@ compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
messaging.frame.size=4096
messaging.frame.count=512
-txn.lock.table.size=1000
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 9e8fb29..cc38ea0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -47,7 +47,7 @@
"txn\.job\.recovery\.memorysize" : 67108864,
"txn\.lock\.escalationthreshold" : 1000,
"txn\.lock\.shrinktimer" : 5000,
- "txn\.lock\.table\.size" : 1000,
+ "txn\.lock\.table\.size" : 1009,
"txn\.lock\.timeout\.sweepthreshold" : 10000,
"txn\.lock\.timeout\.waitthreshold" : 60000,
"txn\.log\.buffer\.numpages" : 8,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 5cded03..e8af59d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -47,7 +47,7 @@
"txn\.job\.recovery\.memorysize" : 67108864,
"txn\.lock\.escalationthreshold" : 1000,
"txn\.lock\.shrinktimer" : 5000,
- "txn\.lock\.table\.size" : 1000,
+ "txn\.lock\.table\.size" : 1009,
"txn\.lock\.timeout\.sweepthreshold" : 10000,
"txn\.lock\.timeout\.waitthreshold" : 60000,
"txn\.log\.buffer\.numpages" : 8,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index e1cbd5d..9d877c9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -47,7 +47,7 @@
"txn\.job\.recovery\.memorysize" : 67108864,
"txn\.lock\.escalationthreshold" : 1000,
"txn\.lock\.shrinktimer" : 5000,
- "txn\.lock\.table\.size" : 1000,
+ "txn\.lock\.table\.size" : 1009,
"txn\.lock\.timeout\.sweepthreshold" : 10000,
"txn\.lock\.timeout\.waitthreshold" : 60000,
"txn\.log\.buffer\.numpages" : 8,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index 3b6a893..a99a306 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -43,9 +43,9 @@ public class StorageProperties extends AbstractProperties {
public enum Option implements IOption {
STORAGE_BUFFERCACHE_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)),
// By default, uses 1/4 of the maximum heap size for read cache, i.e., disk buffer cache.
- STORAGE_BUFFERCACHE_SIZE(LONG_BYTE_UNIT, Runtime.getRuntime().maxMemory() / 4),
+ STORAGE_BUFFERCACHE_SIZE(LONG_BYTE_UNIT, MAX_HEAP_BYTES / 4),
STORAGE_BUFFERCACHE_MAXOPENFILES(NONNEGATIVE_INTEGER, Integer.MAX_VALUE),
- STORAGE_MEMORYCOMPONENT_GLOBALBUDGET(LONG_BYTE_UNIT, Runtime.getRuntime().maxMemory() / 4),
+ STORAGE_MEMORYCOMPONENT_GLOBALBUDGET(LONG_BYTE_UNIT, MAX_HEAP_BYTES / 4),
STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128, KILOBYTE)),
STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(POSITIVE_INTEGER, 2),
STORAGE_MEMORYCOMPONENT_FLUSH_THRESHOLD(DOUBLE, 0.9d),
@@ -132,6 +132,7 @@ public class StorageProperties extends AbstractProperties {
}
}
+ public static final long MAX_HEAP_BYTES = Runtime.getRuntime().maxMemory();
private static final int SYSTEM_RESERVED_DATASETS = 0;
public StorageProperties(PropertiesAccessor accessor) {
@@ -184,13 +185,12 @@ public class StorageProperties extends AbstractProperties {
}
public long getJobExecutionMemoryBudget() {
- final long jobExecutionMemory =
- Runtime.getRuntime().maxMemory() - getBufferCacheSize() - getMemoryComponentGlobalBudget();
+ final long jobExecutionMemory = MAX_HEAP_BYTES - getBufferCacheSize() - getMemoryComponentGlobalBudget();
if (jobExecutionMemory <= 0) {
final String msg = String.format(
"Invalid node memory configuration, more memory budgeted than available in JVM. Runtime max memory:"
+ " (%d), Buffer cache memory (%d), memory component global budget (%d)",
- Runtime.getRuntime().maxMemory(), getBufferCacheSize(), getMemoryComponentGlobalBudget());
+ MAX_HEAP_BYTES, getBufferCacheSize(), getMemoryComponentGlobalBudget());
throw new IllegalStateException(msg);
}
return jobExecutionMemory;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
index b6f2b18..050a493 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java
@@ -27,7 +27,9 @@ import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import org.apache.hyracks.api.config.IApplicationConfig;
import org.apache.hyracks.api.config.IOption;
import org.apache.hyracks.api.config.IOptionType;
import org.apache.hyracks.api.config.Section;
@@ -73,7 +75,13 @@ public class TransactionProperties extends AbstractProperties {
POSITIVE_INTEGER,
10000,
"Interval (in milliseconds) for checking lock " + "timeout"),
- TXN_LOCK_TABLE_SIZE(POSITIVE_INTEGER, 1024 * 1024, "The number of slots in the lock table."),
+ TXN_LOCK_TABLE_SIZE(
+ POSITIVE_INTEGER,
+ TXN_LOCK_TABLE_SIZE_DEFAULT,
+ "The number of slots in the lock table (should be a prime number)",
+ "for JVM max heaps < 8 GB, 1009, otherwise 1048583: (e.g. " + TXN_LOCK_TABLE_SIZE_DEFAULT
+ + " for the current max heap of "
+ + StorageUtil.toHumanReadableSize(StorageProperties.MAX_HEAP_BYTES) + ")"),
TXN_COMMITPROFILER_ENABLED(BOOLEAN, false, "Enable output of commit profiler logs"),
TXN_COMMITPROFILER_REPORTINTERVAL(POSITIVE_INTEGER, 5, "Interval (in seconds) to report commit profiler logs"),
TXN_JOB_RECOVERY_MEMORYSIZE(
@@ -84,11 +92,17 @@ public class TransactionProperties extends AbstractProperties {
private final IOptionType type;
private final Object defaultValue;
private final String description;
+ private final String usageOverride;
Option(IOptionType type, Object defaultValue, String description) {
+ this(type, defaultValue, description, null);
+ }
+
+ Option(IOptionType type, Object defaultValue, String description, String usageOverride) {
this.type = type;
this.defaultValue = defaultValue;
this.description = description;
+ this.usageOverride = usageOverride;
}
@Override
@@ -110,8 +124,16 @@ public class TransactionProperties extends AbstractProperties {
public Object defaultValue() {
return defaultValue;
}
+
+ @Override
+ public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) {
+ return usageOverride;
+ }
}
+ private static final int TXN_LOCK_TABLE_SIZE_DEFAULT =
+ StorageProperties.MAX_HEAP_BYTES < StorageUtil.getByteValue("8GB") ? 1009 : 1048583;
+
public static final String TXN_LOG_PARTITIONSIZE_KEY = Option.TXN_LOG_PARTITIONSIZE.ini();
public static final String TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY = Option.TXN_LOG_CHECKPOINT_POLLFREQUENCY.ini();
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java
index ddab169..bef8847 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ConcurrentLockManager.java
@@ -55,7 +55,7 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
private static final boolean ENABLED_DEADLOCK_FREE_LOCKING_PROTOCOL = true;
private static final int NIL = -1;
- private static final long NILL = -1L;
+ static final long NILL = -1L;
private static final boolean DEBUG_MODE = false;//true
private static final boolean CHECK_CONSISTENCY = false;
@@ -99,7 +99,7 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
public ConcurrentLockManager(final int lockManagerShrinkTimer, final int noArenas, final int tableSize)
throws ACIDException {
- this.table = new ResourceGroupTable(tableSize);
+ table = new ResourceGroupTable(tableSize);
resArenaMgr = new ResourceArenaManager(noArenas, lockManagerShrinkTimer);
reqArenaMgr = new RequestArenaManager(noArenas, lockManagerShrinkTimer);
jobArenaMgr = new JobArenaManager(noArenas, lockManagerShrinkTimer);
@@ -338,7 +338,7 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
final long txnId = txnContext.getTxnId().getId();
final ResourceGroup group = table.get(datasetId.getId(), entityHashValue);
- if (group.firstResourceIndex.get() == NILL) {
+ if (group.firstResourceIndex == NILL) {
validateJob(txnContext);
// if we do not have a resource in the group, we know that the
// resource that we are looking for is not locked
@@ -436,7 +436,7 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
final long txnId = txnContext.getTxnId().getId();
final ResourceGroup group = table.get(datasetId.getId(), entityHashValue);
- if (group.firstResourceIndex.get() == NILL) {
+ if (group.firstResourceIndex == NILL) {
validateJob(txnContext);
// if we do not have a resource in the group, we know that the
// resource that we are looking for is not locked
@@ -508,9 +508,9 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
reqArenaMgr.deallocate(holder);
// deallocate resource or fix max lock mode
if (resourceNotUsed(resource)) {
- long prev = group.firstResourceIndex.get();
+ long prev = group.firstResourceIndex;
if (prev == resource) {
- group.firstResourceIndex.set(resArenaMgr.getNext(resource));
+ group.firstResourceIndex = resArenaMgr.getNext(resource);
} else {
while (resArenaMgr.getNext(prev) != resource) {
prev = resArenaMgr.getNext(prev);
@@ -602,8 +602,8 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
resSlot = resArenaMgr.allocate();
resArenaMgr.setDatasetId(resSlot, dsId);
resArenaMgr.setPkHashVal(resSlot, entityHashValue);
- resArenaMgr.setNext(resSlot, group.firstResourceIndex.get());
- group.firstResourceIndex.set(resSlot);
+ resArenaMgr.setNext(resSlot, group.firstResourceIndex);
+ group.firstResourceIndex = resSlot;
if (DEBUG_MODE) {
LOGGER.trace("new res slot " + TypeUtil.Global.toString(resSlot) + " (" + dsId + ", " + entityHashValue
+ ")");
@@ -677,7 +677,7 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
private long findResourceInGroup(ResourceGroup group, int dsId, int entityHashValue) {
stats.logCounters(LOGGER, LVL, false);
- long resSlot = group.firstResourceIndex.get();
+ long resSlot = group.firstResourceIndex;
while (resSlot != NILL) {
// either we already have a lock on this resource or we have a
// hash collision
@@ -971,7 +971,7 @@ public class ConcurrentLockManager implements ILockManager, ILifeCycleComponent
final ResourceGroup group = table.get(i);
if (group.tryLatch(100, TimeUnit.MILLISECONDS)) {
try {
- long resSlot = group.firstResourceIndex.get();
+ long resSlot = group.firstResourceIndex;
while (resSlot != NILL) {
int dsId = resArenaMgr.getDatasetId(resSlot);
int entityHashValue = resArenaMgr.getPkHashVal(resSlot);
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroup.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroup.java
index 95701d5..44505e7 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroup.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroup.java
@@ -18,10 +18,11 @@
*/
package org.apache.asterix.transaction.management.service.locking;
+import static org.apache.asterix.transaction.management.service.locking.ConcurrentLockManager.NILL;
+
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.asterix.common.transactions.ITransactionContext;
@@ -33,25 +34,25 @@ import org.apache.asterix.common.transactions.ITransactionContext;
* @see ConcurrentLockManager
*/
class ResourceGroup {
- private ReentrantReadWriteLock latch;
- private Condition condition;
- AtomicLong firstResourceIndex;
+ private final ReentrantLock latch;
+ private final Condition condition;
+ volatile long firstResourceIndex;
ResourceGroup() {
- latch = new ReentrantReadWriteLock();
- condition = latch.writeLock().newCondition();
- firstResourceIndex = new AtomicLong(-1);
+ latch = new ReentrantLock();
+ condition = latch.newCondition();
+ firstResourceIndex = NILL;
}
void getLatch() {
log("latch");
- latch.writeLock().lock();
+ latch.lock();
}
boolean tryLatch(long timeout, TimeUnit unit) throws InterruptedException {
log("tryLatch");
try {
- return latch.writeLock().tryLock(timeout, unit);
+ return latch.tryLock(timeout, unit);
} catch (InterruptedException e) {
ConcurrentLockManager.LOGGER.trace("interrupted while wating on ResourceGroup");
throw e;
@@ -60,7 +61,7 @@ class ResourceGroup {
void releaseLatch() {
log("release");
- latch.writeLock().unlock();
+ latch.unlock();
}
boolean hasWaiters() {
@@ -89,7 +90,7 @@ class ResourceGroup {
}
public String toString() {
- return "{ id : " + hashCode() + ", first : " + TypeUtil.Global.toString(firstResourceIndex.get()) + ", "
+ return "{ id : " + hashCode() + ", first : " + TypeUtil.Global.toString(firstResourceIndex) + ", "
+ "waiters : " + (hasWaiters() ? "true" : "false") + " }";
}
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
index 3759f96..93cda83 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceGroupTable.java
@@ -28,9 +28,8 @@ package org.apache.asterix.transaction.management.service.locking;
*/
class ResourceGroupTable {
- public final int size;
-
- private ResourceGroup[] table;
+ final int size;
+ private final ResourceGroup[] table;
public ResourceGroupTable(int size) {
this.size = size;
@@ -42,10 +41,7 @@ class ResourceGroupTable {
ResourceGroup get(int dId, int entityHashValue) {
// TODO ensure good properties of hash function
- int h = Math.abs(dId ^ entityHashValue);
- if (h < 0)
- h = 0;
- return table[h % size];
+ return table[Math.abs(dId ^ entityHashValue) % size];
}
ResourceGroup get(int i) {
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceTablePrinter.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceTablePrinter.java
index e8ef2c8..24691dd 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceTablePrinter.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/locking/ResourceTablePrinter.java
@@ -49,13 +49,13 @@ public class ResourceTablePrinter implements TablePrinter {
int i = 0;
long res = -1;
while (res == -1 && i < table.size) {
- res = table.get(i++).firstResourceIndex.get();
+ res = table.get(i++).firstResourceIndex;
}
while (i < table.size) {
sb = appendResource(sb, res);
res = resArenaMgr.getNext(res);
while (res == -1 && i < table.size) {
- res = table.get(i++).firstResourceIndex.get();
+ res = table.get(i++).firstResourceIndex;
}
if (res == -1) {
sb.append("\n");
[asterixdb] 04/04: Merge branch 'gerrit/cheshire-cat'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0b0d19a5dda1ebaf9bad4c93906dd1056209760b
Merge: 59e3d6a 92ae16e
Author: Michael Blow <mb...@apache.org>
AuthorDate: Tue Dec 22 11:45:35 2020 -0500
Merge branch 'gerrit/cheshire-cat'
Change-Id: I3d754049b8256b3594a483733a6214ff24fc5e27
.../asterix-app/src/main/resources/cc-rep.conf | 3 +--
asterixdb/asterix-app/src/main/resources/cc.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc2.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc3.conf | 1 -
asterixdb/asterix-app/src/main/resources/cc4.conf | 1 -
.../src/test/resources/cc-compression.conf | 1 -
.../src/test/resources/cc-multipart.conf | 1 -
.../asterix-app/src/test/resources/cc-rep-ssl.conf | 3 +--
.../asterix-app/src/test/resources/cc-single.conf | 1 -
.../test/resources/cc-small-txn-log-partition.conf | 1 -
.../asterix-app/src/test/resources/cc-ssl.conf | 3 +--
.../asterix-app/src/test/resources/cc-storage.conf | 3 +--
asterixdb/asterix-app/src/test/resources/cc.conf | 1 -
.../api/cluster_state_1/cluster_state_1.1.regexadm | 2 +-
.../cluster_state_1_full.1.regexadm | 2 +-
.../cluster_state_1_less.1.regexadm | 2 +-
.../asterix/common/config/StorageProperties.java | 10 ++++----
.../common/config/TransactionProperties.java | 24 ++++++++++++++++-
.../service/locking/ConcurrentLockManager.java | 20 +++++++--------
.../management/service/locking/ResourceGroup.java | 25 +++++++++---------
.../service/locking/ResourceGroupTable.java | 10 +++-----
.../service/locking/ResourceTablePrinter.java | 4 +--
.../org/apache/hyracks/api/dataflow/IActivity.java | 22 +++++++++++++---
.../runtime/SuperActivityOperatorNodePushable.java | 4 +--
.../hyracks/api/dataflow/IActivityTest.java} | 30 +++++++++++++++++++---
.../hyracks/control/nc/work/StartTasksWork.java | 5 ++--
.../main/java/org/apache/hyracks/util/Span.java | 14 +++-------
27 files changed, 115 insertions(+), 80 deletions(-)
[asterixdb] 03/04: [NO ISSUE][HYR][UTIL] += Span.ELAPSED;
misc cleanup
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 92ae16ea10fc0d12e443b08a2c4165cb5538742e
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon Dec 21 19:07:43 2020 -0500
[NO ISSUE][HYR][UTIL] += Span.ELAPSED; misc cleanup
Change-Id: Idcf079dd62fd77d0823ccdae4e3961af255f67bb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9423
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../src/main/java/org/apache/hyracks/util/Span.java | 14 +++-----------
1 file changed, 3 insertions(+), 11 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
index a9d8ec8..dc8bc11 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
@@ -70,21 +70,13 @@ public class Span {
}
@Override
- public void loopUntilExhausted(ThrowingAction action) throws Exception {
- super.loopUntilExhausted(action);
- }
-
- @Override
- public void loopUntilExhausted(ThrowingAction action, long delay, TimeUnit delayUnit) throws Exception {
- super.loopUntilExhausted(action, delay, delayUnit);
- }
-
- @Override
public String toString() {
- return "<INFINITY>";
+ return "<INFINITE>";
}
};
+ public static final Span ELAPSED = start(0, TimeUnit.NANOSECONDS);
+
private final long spanNanos;
private volatile long startNanos;