You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2020/09/24 14:36:16 UTC
[hadoop-ozone] branch master updated: HDDS-4270. Add more reusable
byteman scripts to debug ofs/o3fs performance (#1443)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5f1900a HDDS-4270. Add more reusable byteman scripts to debug ofs/o3fs performance (#1443)
5f1900a is described below
commit 5f1900aae0b4d336090aec8c75404efd8634c672
Author: Elek, Márton <el...@users.noreply.github.com>
AuthorDate: Thu Sep 24 16:36:02 2020 +0200
HDDS-4270. Add more reusable byteman scripts to debug ofs/o3fs performance (#1443)
---
dev-support/byteman/appendlog.btm | 28 +++++++++++++
dev-support/byteman/hcfs-read.btm | 67 ++++++++++++++++++++++++++++++
dev-support/byteman/ratis-flush.btm | 34 +++++++++++++++
dev-support/byteman/ratis-no-flush.btm | 25 +++++++++++
dev-support/byteman/watchforcommit.btm | 35 ++++++++++++++++
dev-support/byteman/watchforcommit_all.btm | 47 +++++++++++++++++++++
6 files changed, 236 insertions(+)
diff --git a/dev-support/byteman/appendlog.btm b/dev-support/byteman/appendlog.btm
new file mode 100644
index 0000000..0744bbc
--- /dev/null
+++ b/dev-support/byteman/appendlog.btm
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#Measure time between GrpcLogAppender.appendLog calls (HB frequency)
+
+RULE GrpcLogAppender.appendLog
+CLASS GrpcLogAppender
+METHOD appendLog
+AT ENTRY
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("appendLog" + Thread.currentThread().getId()))
+IF TRUE
+DO
+ System.out.println("appendLog: " + elapsedTime);
+ resetTimer("appendLog" + Thread.currentThread().getId());
+ENDRULE
\ No newline at end of file
diff --git a/dev-support/byteman/hcfs-read.btm b/dev-support/byteman/hcfs-read.btm
new file mode 100644
index 0000000..7200437
--- /dev/null
+++ b/dev-support/byteman/hcfs-read.btm
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Measure time spent in HCFS read implementations
+
+RULE hcfs-read-FileSystem.close
+CLASS org.apache.hadoop.fs.FileSystem
+METHOD close
+IF TRUE
+DO
+ System.out.println("Closing file system instance: " + System.identityHashCode($0));
+ System.out.println(" read.call: " + readCounter("read.call"));
+ System.out.println(" read.allTime: " + readCounter("read.allTime"));
+ System.out.println(" readFully.call: " + readCounter("readFully.call"));
+ System.out.println(" readFully.allTime: " + readCounter("readFully.allTime"));
+ENDRULE
+
+
+RULE FSDataInputStream.Read.Entry
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT ENTRY
+IF TRUE
+DO resetTimer("read" + Thread.currentThread().getId());
+ incrementCounter("read.call")
+ENDRULE
+
+RULE FSDataInputStream.Read.Exit
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("read" + Thread.currentThread().getId()))
+IF TRUE
+DO
+ incrementCounter("read.allTime", elapsedTime)
+ENDRULE
+
+RULE FSDataInputStream.ReadFully.Entry
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT ENTRY
+IF TRUE
+DO resetTimer("readFully" + Thread.currentThread().getId());
+ incrementCounter("readFully.call")
+ENDRULE
+
+RULE FSDataInputStream.ReadFully.Exit
+CLASS org.apache.hadoop.fs.FSDataInputStream
+METHOD read
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("readFully" + Thread.currentThread().getId()))
+IF TRUE
+DO
+ incrementCounter("readFully.allTime", elapsedTime)
+ENDRULE
diff --git a/dev-support/byteman/ratis-flush.btm b/dev-support/byteman/ratis-flush.btm
new file mode 100644
index 0000000..24102a7
--- /dev/null
+++ b/dev-support/byteman/ratis-flush.btm
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Print out time spent in Ratis log flush
+
+RULE SegmentedRaftLogOutputStream.flush.Entry
+CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream
+METHOD flush
+AT ENTRY
+IF TRUE
+DO resetTimer("flush" + Thread.currentThread().getId());
+
+ENDRULE
+
+RULE SegmentedRaftLogOutputStream.flush.Exit
+CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream
+METHOD flush
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("flush" + Thread.currentThread().getId()))
+DO
+ System.out.println("flush: " + elapsedTime);
+ENDRULE
diff --git a/dev-support/byteman/ratis-no-flush.btm b/dev-support/byteman/ratis-no-flush.btm
new file mode 100644
index 0000000..71416d5
--- /dev/null
+++ b/dev-support/byteman/ratis-no-flush.btm
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This btm script turns off the flush when Ratis Log is written
+
+RULE Ratis.noFlush
+CLASS org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogOutputStream
+METHOD flush
+AT ENTRY
+IF TRUE
+DO
+ return
+ENDRULE
diff --git a/dev-support/byteman/watchforcommit.btm b/dev-support/byteman/watchforcommit.btm
new file mode 100644
index 0000000..9b2e70f
--- /dev/null
+++ b/dev-support/byteman/watchforcommit.btm
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Measure time spent in Watch for commit calls
+
+RULE BlockOutputStream.watchForCommit.Entry
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT ENTRY
+IF TRUE
+DO resetTimer("watchForCommit" + Thread.currentThread().getId());
+
+ENDRULE
+
+RULE BlockOutputStream.watchForCommit.Exit
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer("watchForCommit" + Thread.currentThread().getId()))
+IF elapsedTime > 5
+DO
+ System.out.println("WatchForCommit: " + elapsedTime);
+ENDRULE
\ No newline at end of file
diff --git a/dev-support/byteman/watchforcommit_all.btm b/dev-support/byteman/watchforcommit_all.btm
new file mode 100644
index 0000000..8e14af8
--- /dev/null
+++ b/dev-support/byteman/watchforcommit_all.btm
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Measure overall time spent in Watch for commit calls
+
+RULE watchforcommit_all-FileSystem.close
+CLASS org.apache.hadoop.fs.FileSystem
+METHOD close
+IF TRUE
+DO
+ System.out.println("Closing file system instance: " + System.identityHashCode($0));
+ System.out.println(" watchForCommit.call: " + readCounter("watchForCommit.call"));
+ System.out.println(" watchForCommit.allTime: " + readCounter("watchForCommit.allTime"))
+
+ENDRULE
+
+RULE watchforcommit_all-BlockOutputStream.watchForCommit.Entry
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT ENTRY
+IF TRUE
+DO resetTimer(Thread.currentThread());
+ incrementCounter("watchForCommit.call")
+ENDRULE
+
+RULE watchforcommit_all-BlockOutputStream.watchForCommit.Exit
+CLASS org.apache.hadoop.hdds.scm.storage.BlockOutputStream
+METHOD watchForCommit
+AT EXIT
+BIND elapsedTime = java.lang.Math.toIntExact(getElapsedTimeFromTimer(Thread.currentThread()))
+IF TRUE
+DO
+ System.out.println("watchForCommit: " + elapsedTime);
+ incrementCounter("watchForCommit.allTime", elapsedTime)
+ENDRULE
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org