You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ti...@apache.org on 2017/09/28 19:50:31 UTC
asterixdb git commit: [NO ISSUE] Add support for tracing
Repository: asterixdb
Updated Branches:
refs/heads/master f418df3c7 -> b67505d6a
[NO ISSUE] Add support for tracing
- using a format that can can be visualized using chrome://tracing in
Chrome
Change-Id: Ifa83592870e7e38a84aa560e4edb7fa2ad2d5e74
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2022
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b67505d6
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b67505d6
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b67505d6
Branch: refs/heads/master
Commit: b67505d6a707537173833c701f6a86378c1cfc47
Parents: f418df3
Author: Till Westmann <ti...@apache.org>
Authored: Thu Sep 28 07:00:33 2017 -0700
Committer: Till Westmann <ti...@apache.org>
Committed: Thu Sep 28 12:50:09 2017 -0700
----------------------------------------------------------------------
.../api/application/INCServiceContext.java | 8 +
.../control/cc/work/GetNodeDetailsJSONWork.java | 2 +-
.../control/common/controllers/NCConfig.java | 13 +-
.../hyracks/control/common/utils/PidHelper.java | 52 -------
.../control/nc/NodeControllerService.java | 2 +-
.../nc/application/NCServiceContext.java | 14 +-
.../hyracks-storage-am-lsm-btree/pom.xml | 5 +
.../btree/dataflow/LSMBTreeLocalResource.java | 2 +-
.../storage/am/lsm/btree/impls/LSMBTree.java | 5 +-
.../am/lsm/btree/utils/LSMBTreeUtil.java | 5 +-
.../am/lsm/common/impls/AbstractLSMIndex.java | 9 +-
.../lsm/common/impls/AsynchronousScheduler.java | 7 +-
.../am/lsm/common/impls/FlushOperation.java | 16 +-
.../am/lsm/common/impls/TracedIOOperation.java | 96 ++++++++++++
.../invertedindex/impls/LSMInvertedIndex.java | 3 +-
.../am/lsm/rtree/impls/AbstractLSMRTree.java | 3 +-
.../test/support/TestNCServiceContext.java | 6 +
.../hyracks-storage-am-lsm-btree-test/pom.xml | 6 +-
.../am/lsm/btree/LSMBTreeExamplesTest.java | 3 +-
...MBTreeModificationOperationCallbackTest.java | 3 +-
.../LSMBTreeSearchOperationCallbackTest.java | 3 +-
.../am/lsm/btree/LSMBTreeUpdateInPlaceTest.java | 3 +-
.../multithread/LSMBTreeMultiThreadTest.java | 3 +-
.../am/lsm/btree/perf/LSMTreeRunner.java | 3 +-
.../am/lsm/btree/util/LSMBTreeTestContext.java | 7 +-
.../java/org/apache/hyracks/util/PidHelper.java | 52 +++++++
.../org/apache/hyracks/util/trace/Tracer.java | 151 +++++++++++++++++++
27 files changed, 393 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java
index d72cc01..9a484ca 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/INCServiceContext.java
@@ -22,6 +22,7 @@ import org.apache.hyracks.api.comm.IChannelInterfaceFactory;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
import org.apache.hyracks.api.resources.memory.IMemoryManager;
+import org.apache.hyracks.util.trace.Tracer;
/**
* Service Context at the Node Controller for an application.
@@ -54,6 +55,13 @@ public interface INCServiceContext extends IServiceContext {
IMemoryManager getMemoryManager();
/**
+ * Get a Tracer to write trace events to.
+ *
+ * @return a Tracer
+ */
+ Tracer getTracer();
+
+ /**
* Set the handler for state dumps.
*
* @param handler
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java
index 3dec959..6433223 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetNodeDetailsJSONWork.java
@@ -39,7 +39,7 @@ import org.apache.hyracks.control.cc.cluster.INodeManager;
import org.apache.hyracks.control.common.config.ConfigUtils;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.control.common.controllers.NCConfig;
-import org.apache.hyracks.control.common.utils.PidHelper;
+import org.apache.hyracks.util.PidHelper;
import org.apache.hyracks.control.common.work.IPCResponder;
import org.apache.hyracks.control.common.work.SynchronizableWork;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 0fc3be7..02a469d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -78,7 +78,8 @@ public class NCConfig extends ControllerConfig {
APP_CLASS(STRING, (String) null),
NCSERVICE_PID(INTEGER, -1),
COMMAND(STRING, "hyracksnc"),
- JVM_ARGS(STRING, (String) null);
+ JVM_ARGS(STRING, (String) null),
+ TRACE_CATEGORIES(STRING_ARRAY, new String[0]);
private final IOptionType parser;
private final String defaultValueDescription;
@@ -190,6 +191,8 @@ public class NCConfig extends ControllerConfig {
return "Command NCService should invoke to start the NCDriver";
case JVM_ARGS:
return "JVM args to pass to the NCDriver";
+ case TRACE_CATEGORIES:
+ return "Categories for tracing";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -445,6 +448,14 @@ public class NCConfig extends ControllerConfig {
configManager.set(nodeId, Option.IODEVICES, iodevices);
}
+ public String[] getTraceCategories() {
+ return appConfig.getStringArray(Option.TRACE_CATEGORIES);
+ }
+
+ public void setTraceCategories(String[] traceCategories) {
+ configManager.set(nodeId, Option.TRACE_CATEGORIES, traceCategories);
+ }
+
public int getNetThreadCount() {
return appConfig.getInt(Option.NET_THREAD_COUNT);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/PidHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/PidHelper.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/PidHelper.java
deleted file mode 100644
index 79642c0..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/PidHelper.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.common.utils;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class PidHelper {
-
- private static final Logger LOGGER = Logger.getLogger(PidHelper.class.getName());
-
- private PidHelper() {
- }
-
- public static int getPid() {
- return getPid(ManagementFactory.getRuntimeMXBean());
- }
-
- public static int getPid(RuntimeMXBean runtimeMXBean) {
- try {
- Field jvmField = runtimeMXBean.getClass().getDeclaredField("jvm");
- jvmField.setAccessible(true);
- Object vmManagement = jvmField.get(runtimeMXBean);
- Method getProcessIdMethod = vmManagement.getClass().getDeclaredMethod("getProcessId");
- getProcessIdMethod.setAccessible(true);
- return (Integer) getProcessIdMethod.invoke(vmManagement);
- } catch (Exception e) {
- LOGGER.log(Level.INFO, "Unable to determine PID due to exception", e);
- return -1;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index a426d47..b0a702d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -69,7 +69,7 @@ import org.apache.hyracks.control.common.ipc.CCNCFunctions;
import org.apache.hyracks.control.common.ipc.ClusterControllerRemoteProxy;
import org.apache.hyracks.control.common.ipc.IControllerRemoteProxyIPCEventListener;
import org.apache.hyracks.control.common.job.profiling.om.JobProfile;
-import org.apache.hyracks.control.common.utils.PidHelper;
+import org.apache.hyracks.util.PidHelper;
import org.apache.hyracks.control.common.work.FutureValue;
import org.apache.hyracks.control.common.work.WorkQueue;
import org.apache.hyracks.control.nc.application.NCServiceContext;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java
index d659fe6..d1d2dc9 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/application/NCServiceContext.java
@@ -34,6 +34,7 @@ import org.apache.hyracks.control.common.utils.HyracksThreadFactory;
import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.hyracks.control.nc.resources.memory.MemoryManager;
+import org.apache.hyracks.util.trace.Tracer;
public class NCServiceContext extends ServiceContext implements INCServiceContext {
private final ILifeCycleComponentManager lccm;
@@ -43,9 +44,10 @@ public class NCServiceContext extends ServiceContext implements INCServiceContex
private IStateDumpHandler sdh;
private final NodeControllerService ncs;
private IChannelInterfaceFactory messagingChannelInterfaceFactory;
+ private final Tracer tracer;
- public NCServiceContext(NodeControllerService ncs, ServerContext serverCtx, IOManager ioManager,
- String nodeId, MemoryManager memoryManager, ILifeCycleComponentManager lifeCyclecomponentManager,
+ public NCServiceContext(NodeControllerService ncs, ServerContext serverCtx, IOManager ioManager, String nodeId,
+ MemoryManager memoryManager, ILifeCycleComponentManager lifeCyclecomponentManager,
IApplicationConfig appConfig) throws IOException {
super(serverCtx, appConfig, new HyracksThreadFactory(nodeId));
this.lccm = lifeCyclecomponentManager;
@@ -53,7 +55,8 @@ public class NCServiceContext extends ServiceContext implements INCServiceContex
this.ioManager = ioManager;
this.memoryManager = memoryManager;
this.ncs = ncs;
- sdh = lccm::dumpState;
+ this.sdh = lccm::dumpState;
+ this.tracer = new Tracer(nodeId, ncs.getConfiguration().getTraceCategories());
}
@Override
@@ -90,6 +93,11 @@ public class NCServiceContext extends ServiceContext implements INCServiceContex
}
@Override
+ public Tracer getTracer() {
+ return tracer;
+ }
+
+ @Override
public IControllerService getControllerService() {
return ncs;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
index b3798ab..48961db 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/pom.xml
@@ -86,5 +86,10 @@
<artifactId>hyracks-data-std</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
index c283502..dad56eb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
@@ -77,6 +77,6 @@ public class LSMBTreeLocalResource extends LsmResource {
mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory.createIoOpCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields,
- filterFields, durable, metadataPageManagerFactory, updateAware);
+ filterFields, durable, metadataPageManagerFactory, updateAware, serviceCtx.getTracer());
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 24b408c..1920592 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -76,6 +76,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.Tracer;
public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
@@ -106,10 +107,10 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
- boolean durable, boolean updateAware) throws HyracksDataException {
+ boolean durable, boolean updateAware, Tracer tracer) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
- filterManager, filterFields, durable, filterHelper, btreeFields);
+ filterManager, filterFields, durable, filterHelper, btreeFields, tracer);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
index 5e50eee..a9c7a80 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
@@ -53,6 +53,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.Tracer;
public class LSMBTreeUtil {
@@ -65,7 +66,7 @@ public class LSMBTreeUtil {
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable,
- IMetadataPageManagerFactory freePageManagerFactory, boolean updateAware)
+ IMetadataPageManagerFactory freePageManagerFactory, boolean updateAware, Tracer tracer)
throws HyracksDataException {
LSMBTreeTupleWriterFactory insertTupleWriterFactory =
new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false, updateAware);
@@ -109,7 +110,7 @@ public class LSMBTreeUtil {
deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length,
cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields,
- filterFields, durable, updateAware);
+ filterFields, durable, updateAware, tracer);
}
public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index c5bf2ca..beed6b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -63,6 +63,7 @@ import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.Tracer;
public abstract class AbstractLSMIndex implements ILSMIndex {
protected final ILSMHarness lsmHarness;
@@ -90,13 +91,14 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
protected boolean isActive;
protected final AtomicBoolean[] flushRequests;
protected boolean memoryComponentsAllocated = false;
+ protected Tracer tracer;
public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
LSMComponentFilterManager filterManager, int[] filterFields, boolean durable,
- IComponentFilterHelper filterHelper, int[] treeFields) {
+ IComponentFilterHelper filterHelper, int[] treeFields, Tracer tracer) {
this.ioManager = ioManager;
this.virtualBufferCaches = virtualBufferCaches;
this.diskBufferCache = diskBufferCache;
@@ -112,6 +114,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
this.filterFields = filterFields;
this.inactiveDiskComponents = new LinkedList<>();
this.durable = durable;
+ this.tracer = tracer;
lsmHarness = new LSMHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled());
isActive = false;
diskComponents = new ArrayList<>();
@@ -333,7 +336,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
opCtx.setOperation(IndexOperation.FLUSH);
opCtx.getComponentHolder().add(flushingComponent);
ILSMIOOperation flushOp = createFlushOperation(opCtx, flushingComponent, componentFileRefs, callback);
- ioScheduler.scheduleOperation(flushOp);
+ ioScheduler.scheduleOperation(TracedIOOperation.wrap(flushOp, tracer));
}
@Override
@@ -348,7 +351,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent);
ILSMIOOperation mergeOp = createMergeOperation(opCtx, mergingComponents, mergeFileRefs, callback);
- ioScheduler.scheduleOperation(mergeOp);
+ ioScheduler.scheduleOperation(TracedIOOperation.wrap(mergeOp, tracer));
}
private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
index 50e240e..5f6766f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AsynchronousScheduler.java
@@ -41,12 +41,13 @@ public class AsynchronousScheduler implements ILSMIOOperationScheduler {
public final static AsynchronousScheduler INSTANCE = new AsynchronousScheduler();
private ExecutorService executor;
private final Map<String, ILSMIOOperation> runningFlushOperations = new HashMap<String, ILSMIOOperation>();
- private final Map<String, PriorityQueue<ILSMIOOperation>> waitingFlushOperations = new HashMap<String, PriorityQueue<ILSMIOOperation>>();
+ private final Map<String, PriorityQueue<ILSMIOOperation>> waitingFlushOperations =
+ new HashMap<String, PriorityQueue<ILSMIOOperation>>();
public void init(ThreadFactory threadFactory) {
// Creating an executor with the same configuration of Executors.newCachedThreadPool.
- executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(), threadFactory) {
+ executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ threadFactory) {
@Override
protected <T> RunnableFuture<T> newTaskFor(Callable<T> callable) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
index cafb008..3d65a6f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/FlushOperation.java
@@ -23,11 +23,12 @@ import java.util.Objects;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
-public class FlushOperation extends AbstractIoOperation implements Comparable<FlushOperation> {
+public class FlushOperation extends AbstractIoOperation implements Comparable<ILSMIOOperation> {
protected final ILSMMemoryComponent flushingComponent;
@@ -73,16 +74,17 @@ public class FlushOperation extends AbstractIoOperation implements Comparable<Fl
}
@Override
- public int compareTo(FlushOperation o) {
- return target.getFile().getName().compareTo(o.getTarget().getFile().getName());
+ public int compareTo(ILSMIOOperation o) {
+ if (o instanceof FlushOperation) {
+ return target.getFile().getName().compareTo(((FlushOperation) o).getTarget().getFile().getName());
+ }
+ return -1;
}
@Override
public boolean equals(Object o) {
- if (!(o instanceof FlushOperation)) {
- return false;
- }
- return Objects.equals(target.getFile().getName(), ((FlushOperation) o).target.getFile().getName());
+ return (o instanceof FlushOperation)
+ && Objects.equals(target.getFile().getName(), ((FlushOperation) o).target.getFile().getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
new file mode 100644
index 0000000..a58b0b1
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.util.trace.Tracer;
+
+class TracedIOOperation implements ILSMIOOperation {
+ protected final ILSMIOOperation ioOp;
+ private final LSMIOOpertionType ioOpType;
+ private final Tracer tracer;
+ private final String cat;
+
+ protected TracedIOOperation(ILSMIOOperation ioOp, Tracer tracer) {
+ this.ioOp = ioOp;
+ this.tracer = tracer;
+ this.ioOpType = ioOp.getIOOpertionType();
+ this.cat = ioOpType.name().toLowerCase();
+ }
+
+ public static ILSMIOOperation wrap(final ILSMIOOperation ioOp, final Tracer tracer) {
+ if (tracer != null && tracer.isEnabled()) {
+ return ioOp instanceof Comparable ? new ComparableTracedIOOperation(ioOp, tracer)
+ : new TracedIOOperation(ioOp, tracer);
+ }
+ return ioOp;
+ }
+
+ protected ILSMIOOperation getIoOp() {
+ return ioOp;
+ }
+
+ public IODeviceHandle getDevice() {
+ return ioOp.getDevice();
+ }
+
+ public ILSMIOOperationCallback getCallback() {
+ return ioOp.getCallback();
+ }
+
+ public String getIndexIdentifier() {
+ return ioOp.getIndexIdentifier();
+ }
+
+ public LSMIOOpertionType getIOOpertionType() {
+ return ioOpType;
+ }
+
+ @Override
+ public Boolean call() throws HyracksDataException {
+ final long tid = tracer.durationB(getDevice().toString(), cat, null);
+ try {
+ return ioOp.call();
+ } finally {
+ tracer.durationE(tid, "{\"optional\":\"value\"}");
+ }
+ }
+}
+
+class ComparableTracedIOOperation extends TracedIOOperation implements Comparable<ILSMIOOperation> {
+
+ protected ComparableTracedIOOperation(ILSMIOOperation ioOp, Tracer trace) {
+ super(ioOp, trace);
+ System.err.println("COMPARE ComparableTracedIOOperation");
+ }
+
+ public int compareTo(ILSMIOOperation other) {
+ System.err.println("COMPARE compareTo " + other.getClass().getSimpleName());
+ if (other instanceof ComparableTracedIOOperation) {
+ other = ((ComparableTracedIOOperation) other).getIoOp();
+ return ((Comparable) this.ioOp).compareTo(other);
+ }
+ throw new IllegalArgumentException("Comparing ioOps of type " + this.ioOp.getClass().getSimpleName() + " and "
+ + other.getClass().getSimpleName());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 1f2a810..7994bf0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -86,6 +86,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.Tracer;
public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex {
private static final Logger LOGGER = Logger.getLogger(LSMInvertedIndex.class.getName());
@@ -117,7 +118,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
- filterManager, filterFields, durable, filterHelper, invertedIndexFields);
+ filterManager, filterFields, durable, filterHelper, invertedIndexFields, Tracer.none());
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 6ca48b3..eb03696 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -57,6 +57,7 @@ import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.Tracer;
public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITreeIndex {
@@ -91,7 +92,7 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
boolean isPointMBR, IBufferCache diskBufferCache) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable,
- filterHelper, rtreeFields);
+ filterHelper, rtreeFields, Tracer.none());
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
RTree memRTree = new RTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache),
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java
index ee74b75..cbdb7b7 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestNCServiceContext.java
@@ -32,6 +32,7 @@ import org.apache.hyracks.api.lifecycle.LifeCycleComponentManager;
import org.apache.hyracks.api.messages.IMessageBroker;
import org.apache.hyracks.api.resources.memory.IMemoryManager;
import org.apache.hyracks.api.service.IControllerService;
+import org.apache.hyracks.util.trace.Tracer;
public class TestNCServiceContext implements INCServiceContext {
private final ILifeCycleComponentManager lccm;
@@ -86,6 +87,11 @@ public class TestNCServiceContext implements INCServiceContext {
}
@Override
+ public Tracer getTracer() {
+ return Tracer.none();
+ }
+
+ @Override
public void setMessageBroker(IMessageBroker staticticsConnector) {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
index e3f49c7..55c5710 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
@@ -104,9 +104,13 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.5</version>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
index 39b16c6..56af40d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeExamplesTest.java
@@ -37,6 +37,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
import org.apache.hyracks.storage.am.lsm.btree.utils.LSMBTreeUtil;
import org.apache.hyracks.storage.common.IIndexAccessor;
+import org.apache.hyracks.util.trace.Tracer;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -53,7 +54,7 @@ public class LSMBTreeExamplesTest extends OrderedIndexExamplesTest {
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true,
filterTypeTraits, filterCmpFactories, btreeFields, filterFields, true,
- harness.getMetadataPageManagerFactory(), false);
+ harness.getMetadataPageManagerFactory(), false, Tracer.none());
}
@Before
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
index 98672e7..87ccde7 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeModificationOperationCallbackTest.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpOperationTrackerFactory;
+import org.apache.hyracks.util.trace.Tracer;
import org.junit.Test;
public class LSMBTreeModificationOperationCallbackTest extends AbstractModificationOperationCallbackTest {
@@ -52,7 +53,7 @@ public class LSMBTreeModificationOperationCallbackTest extends AbstractModificat
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
harness.getIOOperationCallback(), true, null, null, null, null, true,
- harness.getMetadataPageManagerFactory(), false);
+ harness.getMetadataPageManagerFactory(), false, Tracer.none());
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index 8aa4428..73497a9 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -38,6 +38,7 @@ import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.hyracks.util.trace.Tracer;
import org.junit.Assert;
import org.junit.Test;
@@ -58,7 +59,7 @@ public class LSMBTreeSearchOperationCallbackTest extends AbstractSearchOperation
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
harness.getIOOperationCallback(), true, null, null, null, null, true,
- harness.getMetadataPageManagerFactory(), false);
+ harness.getMetadataPageManagerFactory(), false, Tracer.none());
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java
index 4cfb102..2e68e44 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeUpdateInPlaceTest.java
@@ -39,6 +39,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFac
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpOperationTrackerFactory;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
+import org.apache.hyracks.util.trace.Tracer;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -74,7 +75,7 @@ public class LSMBTreeUpdateInPlaceTest extends AbstractOperationCallbackTest {
harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
NoOpOperationTrackerFactory.INSTANCE.getOperationTracker(null), harness.getIOScheduler(),
harness.getIOOperationCallback(), true, null, null, null, null, true,
- harness.getMetadataPageManagerFactory(), true);
+ harness.getMetadataPageManagerFactory(), true, Tracer.all());
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
index db626c8..4ebbe5f 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeMultiThreadTest.java
@@ -32,6 +32,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.datagen.ProbabilityHelper;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
import org.apache.hyracks.storage.am.lsm.btree.utils.LSMBTreeUtil;
+import org.apache.hyracks.util.trace.Tracer;
public class LSMBTreeMultiThreadTest extends OrderedIndexMultiThreadTest {
@@ -56,7 +57,7 @@ public class LSMBTreeMultiThreadTest extends OrderedIndexMultiThreadTest {
harness.getFileReference(), harness.getDiskBufferCache(), typeTraits, cmpFactories,
bloomFilterKeyFields, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallback(), true, null,
- null, null, null, true, harness.getMetadataPageManagerFactory(), false);
+ null, null, null, true, harness.getMetadataPageManagerFactory(), false, Tracer.none());
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
index a7ed646..50078ef 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/LSMTreeRunner.java
@@ -48,6 +48,7 @@ import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
+import org.apache.hyracks.util.trace.Tracer;
public class LSMTreeRunner implements IExperimentRunner {
@@ -109,7 +110,7 @@ public class LSMTreeRunner implements IExperimentRunner {
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, new NoMergePolicy(),
new ThreadCountingTracker(), ioScheduler, NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(),
true, null, null, null, null, true, TestStorageManagerComponentHolder.getMetadataPageManagerFactory(),
- false);
+ false, Tracer.none());
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
index ec78f6a..6c95f4d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeTestContext.java
@@ -41,6 +41,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.Tracer;
@SuppressWarnings("rawtypes")
public final class LSMBTreeTestContext extends OrderedIndexTestContext {
@@ -95,14 +96,12 @@ public final class LSMBTreeTestContext extends OrderedIndexTestContext {
lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallback, needKeyDupCheck, filterTypeTraits, filterCmp, btreefields, filterfields,
- true,
- metadataPageManagerFactory, updateAware);
+ true, metadataPageManagerFactory, updateAware, Tracer.none());
} else {
lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
ioScheduler, ioOpCallback, needKeyDupCheck, null, null, null, null, true,
- metadataPageManagerFactory,
- updateAware);
+ metadataPageManagerFactory, updateAware, Tracer.all());
}
LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered);
return testCtx;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
new file mode 100644
index 0000000..410097e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class PidHelper {
+
+ private static final Logger LOGGER = Logger.getLogger(PidHelper.class.getName());
+
+ private PidHelper() {
+ }
+
+ public static int getPid() {
+ return getPid(ManagementFactory.getRuntimeMXBean());
+ }
+
+ public static int getPid(RuntimeMXBean runtimeMXBean) {
+ try {
+ Field jvmField = runtimeMXBean.getClass().getDeclaredField("jvm");
+ jvmField.setAccessible(true);
+ Object vmManagement = jvmField.get(runtimeMXBean);
+ Method getProcessIdMethod = vmManagement.getClass().getDeclaredMethod("getProcessId");
+ getProcessIdMethod.setAccessible(true);
+ return (Integer) getProcessIdMethod.invoke(vmManagement);
+ } catch (Exception e) {
+ LOGGER.log(Level.INFO, "Unable to determine PID due to exception", e);
+ return -1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b67505d6/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
new file mode 100644
index 0000000..1c02ac8
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.util.trace;
+
+import java.util.Arrays;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.hyracks.util.PidHelper;
+
+/**
+ * https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/edit
+ */
+public class Tracer {
+
+ protected static final Level TRACE_LOG_LEVEL = Level.INFO;
+
+ protected final Logger traceLog;
+ protected String[] categories;
+
+ protected static final int pid = PidHelper.getPid();
+
+ public enum Phase {
+ // Duration Events
+ B, // begin
+ E, // end
+ // Complete Events
+ X,
+ // Instant Events
+ I,
+ // Counter Events
+ C,
+ // Async Events
+ b, // nestable start
+ n, // nestable instant
+ e, // nestable end
+ // Flow Events
+ s, // start
+ t, // step
+ f, // end
+ // Object Events
+ N, // created
+ O, // snapshot
+ D // destroyed
+ }
+
+ public Tracer(String name, String[] categories) {
+ this.traceLog = Logger.getLogger(Tracer.class.getName() + "@" + name);
+ this.categories = categories;
+ }
+
+ public static Tracer none() {
+ return new Tracer("None", new String[0]);
+ }
+
+ public static Tracer all() {
+ return new Tracer("All", new String[] { "*" });
+ }
+
+ @Override
+ public String toString() {
+ return getName() + Arrays.toString(categories) + (isEnabled() ? "enabled" : "disabled");
+ }
+
+ public String getName() {
+ return traceLog.getName();
+ }
+
+ public boolean isEnabled() {
+ return categories.length > 0;
+ }
+
+ public long durationB(String name, String cat, String args) {
+ Event e = Event.create(name, cat, Phase.B, pid, Thread.currentThread().getId(), args);
+ traceLog.log(TRACE_LOG_LEVEL, e.toJson());
+ return e.tid;
+ }
+
+ public void durationE(long tid, String args) {
+ Event e = Event.create(null, null, Phase.E, pid, tid, args);
+ traceLog.log(TRACE_LOG_LEVEL, e.toJson());
+ }
+}
+
+class Event {
+ public final String name;
+ public final String cat;
+ public final Tracer.Phase ph;
+ public final long ts;
+ public final int pid;
+ public final long tid;
+ public final String args;
+
+ private Event(String name, String cat, Tracer.Phase ph, long ts, int pid, long tid, String args) {
+ this.name = name;
+ this.cat = cat;
+ this.ph = ph;
+ this.ts = ts;
+ this.pid = pid;
+ this.tid = tid;
+ this.args = args;
+ }
+
+ private static long timestamp() {
+ return System.nanoTime() / 1000;
+ }
+
+ public static Event create(String name, String cat, Tracer.Phase ph, int pid, long tid, String args) {
+ return new Event(name, cat, ph, timestamp(), pid, tid, args);
+ }
+
+ public String toJson() {
+ return append(new StringBuilder()).toString();
+ }
+
+ public StringBuilder append(StringBuilder sb) {
+ sb.append("{");
+ if (name != null) {
+ sb.append("\"name\":\"").append(name).append("\",");
+ }
+ if (cat != null) {
+ sb.append("\"cat\":\"").append(cat).append("\",");
+ }
+ sb.append("\"ph\":\"").append(ph).append("\",");
+ sb.append("\"pid\":\"").append(pid).append("\",");
+ sb.append("\"tid\":").append(tid).append(",");
+ sb.append("\"ts\":").append(ts);
+ if (args != null) {
+ sb.append(",\"args\":").append(args);
+ }
+ sb.append("}");
+ return sb;
+ }
+}